2024-09-21T09:11:55.817 INFO:root:teuthology version: 1.2.2.dev2+g3752d38 2024-09-21T09:11:55.837 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-21T09:11:55.922 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi/7913639 branch: main description: upgrade/cephfs/nofs/{bluestore-bitmap centos_9.stream conf/{client mds mgr mon osd} no-mds-cluster overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn pg_health} tasks/{0-from/reef 1-upgrade}} email: ceph-qa@ceph.com first_in_suite: false job_id: '7913639' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'install ceph/reef latest ' name: teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi no_nested_subset: false openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: main ceph: cluster-conf: client: client mount timeout: 600 debug client: 20 debug ms: 1 rados mon op timeout: 900 rados osd op timeout: 900 mds: debug mds: 20 debug mds balancer: 20 debug ms: 2 mds debug frag: true mds debug scatterstat: true mds op complaint time: 180 mds verify scatter: true osd op complaint time: 180 rados mon op timeout: 900 rados osd op timeout: 900 mon: mon op complaint time: 120 mon warn on pool no app: false osd: osd op complaint time: 180 conf: global: bluestore warn on legacy statfs: false bluestore warn on no per pool omap: false mon pg warn min per osd: 0 mgr: client mount timeout: 30 debug client: 20 debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 mon down mkfs grace: 300 mon warn on osd down out interval zero: false osd: bdev async discard: true bdev enable discard: true bluestore allocator: bitmap bluestore block size: 96636764160 bluestore fsck on mount: true debug bluefs: 1/20 debug bluestore: 1/20 debug ms: 1 debug osd: 20 debug rocksdb: 4/10 mon osd backfillfull_ratio: 0.85 mon osd full ratio: 0.9 mon osd nearfull ratio: 0.8 osd failsafe full ratio: 0.95 osd objectstore: bluestore flavor: default fs: xfs log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - FS_DEGRADED - fs.*is degraded - filesystem is degraded - FS_INLINE_DATA_DEPRECATED - FS_WITH_FAILED_MDS - MDS_ALL_DOWN - filesystem is offline - MDS_DAMAGE - MDS_DEGRADED - MDS_FAILED - MDS_INSUFFICIENT_STANDBY - insufficient standby MDS daemons available - MDS_UP_LESS_THAN_MAX - online, but wants - filesystem is online with fewer MDS than max_mds - POOL_APP_NOT_ENABLED - do not have an application enabled - overall HEALTH_ - Replacing daemon - deprecated feature inline_data - overall HEALTH_ - \(OSD_DOWN\) - \(OSD_ - but it is still running - is not responding - PG_AVAILABILITY - PG_DEGRADED - Reduced data availability - Degraded data redundancy - scrub mismatch - ScrubResult - wrongly marked - \(POOL_APP_NOT_ENABLED\) - \(SLOW_OPS\) - overall HEALTH_ - \(MON_MSGR2_NOT_ENABLED\) - slow request sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 ceph-deploy: bluestore: true conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} osd: bdev async discard: true bdev enable discard: true bluestore block size: 96636764160 bluestore fsck on mount: true debug bluefs: 1/20 debug bluestore: 1/20 debug rocksdb: 4/10 mon osd backfillfull_ratio: 0.85 mon osd full ratio: 0.9 mon osd nearfull ratio: 0.8 osd failsafe full ratio: 0.95 osd objectstore: bluestore fs: xfs install: ceph: flavor: default sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 thrashosds: bdev_inject_crash: 2 bdev_inject_crash_probability: 0.5 workunit: branch: main sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 owner: scheduled_teuthology@teuthology priority: 850 repo: https://git.ceph.com/ceph.git roles: - - mon.a - mon.b - mon.c - mgr.x - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 seed: 5357 sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 sleep_before_teardown: 0 subset: 31/32 suite: upgrade suite_branch: main suite_path: /home/teuthworker/src/git.ceph.com_ceph_62da54b4b83a96ca9976b7e1d0c8272a564a2208/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 targets: smithi078.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBOL5fG+Lq2YIfK3G2HmCT3mD3yizsUrqLk2UWnY/YmsJijyt8gwZ2Waqg5sm3mvb51Gx3FcxoB6m0JboN8aTUNw= tasks: - install: branch: reef exclude_packages: - librados3 - ceph-mgr-dashboard - ceph-mgr-diskprediction-local - ceph-mgr-rook - ceph-mgr-cephadm - cephadm - ceph-volume extra_packages: - librados2 - print: '**** done installing reef' - ceph: conf: global: mon warn on pool no app: false ms bind msgr2: false log-ignorelist: - overall HEALTH_ - \(FS_ - \(MDS_ - \(OSD_ - \(MON_DOWN\) - \(CACHE_POOL_ - \(POOL_ - \(MGR_DOWN\) - \(PG_ - \(SMALLER_PGP_NUM\) - Monitor daemon marked osd - Behind on trimming - Manager daemon - exec: osd.0: - ceph osd set-require-min-compat-client reef - print: '**** done ceph' - print: '*** upgrading, no cephfs present' - exec: mon.a: - ceph fs dump - install.upgrade: mon.a: null - print: '**** done install.upgrade' - ceph.restart: daemons: - mon.* - mgr.* mon-health-to-clog: false wait-for-healthy: false - ceph.healthy: null - ceph.restart: daemons: - osd.* wait-for-healthy: false wait-for-osds-up: true - exec: mon.a: - ceph versions - ceph osd dump -f json-pretty - ceph fs dump - ceph osd require-osd-release squid - for f in `ceph osd pool ls` ; do ceph osd pool set $f pg_autoscale_mode off ; done - ceph.healthy: null - print: '**** done ceph.restart' teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a timestamp: 2024-09-21_03:08:26 tube: smithi user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2902 2024-09-21T09:11:55.923 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_62da54b4b83a96ca9976b7e1d0c8272a564a2208/qa; will attempt to use it 2024-09-21T09:11:55.923 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_62da54b4b83a96ca9976b7e1d0c8272a564a2208/qa/tasks 2024-09-21T09:11:55.923 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-09-21T09:11:55.925 INFO:teuthology.task.internal:Checking packages... 2024-09-21T09:11:55.946 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '62da54b4b83a96ca9976b7e1d0c8272a564a2208' 2024-09-21T09:11:55.946 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-21T09:11:55.946 INFO:teuthology.packaging:ref: None 2024-09-21T09:11:55.946 INFO:teuthology.packaging:tag: None 2024-09-21T09:11:55.946 INFO:teuthology.packaging:branch: main 2024-09-21T09:11:55.947 INFO:teuthology.packaging:sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:11:55.947 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=main 2024-09-21T09:11:56.195 INFO:teuthology.task.internal:Found packages for ceph version 19.3.0-5092.g62da54b4 2024-09-21T09:11:56.197 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-09-21T09:11:56.203 INFO:teuthology.task.internal:no buildpackages task found 2024-09-21T09:11:56.203 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-09-21T09:11:56.227 INFO:teuthology.task.internal:Saving configuration 2024-09-21T09:11:56.243 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-09-21T09:11:56.249 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-09-21T09:11:56.274 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi078.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi/7913639', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-21 09:07:03.552936', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBOL5fG+Lq2YIfK3G2HmCT3mD3yizsUrqLk2UWnY/YmsJijyt8gwZ2Waqg5sm3mvb51Gx3FcxoB6m0JboN8aTUNw='} 2024-09-21T09:11:56.274 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-09-21T09:11:56.280 INFO:teuthology.task.internal:roles: ubuntu@smithi078.front.sepia.ceph.com - ['mon.a', 'mon.b', 'mon.c', 'mgr.x', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3'] 2024-09-21T09:11:56.280 INFO:teuthology.run_tasks:Running task console_log... 2024-09-21T09:11:56.313 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f7c9e5dc160>, signals=[15]) 2024-09-21T09:11:56.313 INFO:teuthology.run_tasks:Running task internal.connect... 2024-09-21T09:11:56.320 INFO:teuthology.task.internal:Opening connections... 2024-09-21T09:11:56.320 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi078.front.sepia.ceph.com 2024-09-21T09:11:56.321 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi078.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T09:11:56.398 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-09-21T09:11:56.404 DEBUG:teuthology.orchestra.run.smithi078:> uname -m 2024-09-21T09:11:56.484 INFO:teuthology.orchestra.run.smithi078.stdout:x86_64 2024-09-21T09:11:56.485 DEBUG:teuthology.orchestra.run.smithi078:> cat /etc/os-release 2024-09-21T09:11:56.542 INFO:teuthology.orchestra.run.smithi078.stdout:NAME="CentOS Stream" 2024-09-21T09:11:56.542 INFO:teuthology.orchestra.run.smithi078.stdout:VERSION="9" 2024-09-21T09:11:56.542 INFO:teuthology.orchestra.run.smithi078.stdout:ID="centos" 2024-09-21T09:11:56.543 INFO:teuthology.orchestra.run.smithi078.stdout:ID_LIKE="rhel fedora" 2024-09-21T09:11:56.543 INFO:teuthology.orchestra.run.smithi078.stdout:VERSION_ID="9" 2024-09-21T09:11:56.543 INFO:teuthology.orchestra.run.smithi078.stdout:PLATFORM_ID="platform:el9" 2024-09-21T09:11:56.543 INFO:teuthology.orchestra.run.smithi078.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-21T09:11:56.543 INFO:teuthology.orchestra.run.smithi078.stdout:ANSI_COLOR="0;31" 2024-09-21T09:11:56.543 INFO:teuthology.orchestra.run.smithi078.stdout:LOGO="fedora-logo-icon" 2024-09-21T09:11:56.543 INFO:teuthology.orchestra.run.smithi078.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-21T09:11:56.543 INFO:teuthology.orchestra.run.smithi078.stdout:HOME_URL="https://centos.org/" 2024-09-21T09:11:56.543 INFO:teuthology.orchestra.run.smithi078.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-21T09:11:56.543 INFO:teuthology.orchestra.run.smithi078.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-21T09:11:56.543 INFO:teuthology.orchestra.run.smithi078.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-21T09:11:56.544 INFO:teuthology.lock.ops:Updating smithi078.front.sepia.ceph.com on lock server 2024-09-21T09:11:56.561 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-09-21T09:11:56.568 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-09-21T09:11:56.573 INFO:teuthology.task.internal:Checking for old test directory... 2024-09-21T09:11:56.573 DEBUG:teuthology.orchestra.run.smithi078:> test '!' -e /home/ubuntu/cephtest 2024-09-21T09:11:56.599 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-09-21T09:11:56.604 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-09-21T09:11:56.604 DEBUG:teuthology.orchestra.run.smithi078:> test -z $(ls -A /var/lib/ceph) 2024-09-21T09:11:56.894 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-09-21T09:11:56.926 INFO:teuthology.run_tasks:Running task kernel... 2024-09-21T09:11:56.940 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-09-21T09:11:56.940 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-09-21T09:11:56.941 DEBUG:teuthology.orchestra.run.smithi078:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-21T09:11:56.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:11:56.958 DEBUG:teuthology.orchestra.run.smithi078:> uname -r 2024-09-21T09:11:57.015 INFO:teuthology.orchestra.run.smithi078.stdout:5.14.0-503.el9.x86_64 2024-09-21T09:11:57.016 INFO:teuthology.task.kernel:Running kernel on smithi078: 5.14.0-503.el9.x86_64 2024-09-21T09:11:57.016 DEBUG:teuthology.orchestra.run.smithi078:> sudo yum install -y kernel 2024-09-21T09:12:01.287 INFO:teuthology.orchestra.run.smithi078.stdout:CentOS Stream 9 - BaseOS 3.2 MB/s | 8.3 MB 00:02 2024-09-21T09:12:04.605 INFO:teuthology.orchestra.run.smithi078.stdout:CentOS Stream 9 - AppStream 9.1 MB/s | 20 MB 00:02 2024-09-21T09:12:09.239 INFO:teuthology.orchestra.run.smithi078.stdout:CentOS Stream 9 - CRB 7.8 MB/s | 6.5 MB 00:00 2024-09-21T09:12:10.681 INFO:teuthology.orchestra.run.smithi078.stdout:CentOS Stream 9 - Extras packages 83 kB/s | 19 kB 00:00 2024-09-21T09:12:11.486 INFO:teuthology.orchestra.run.smithi078.stdout:Extra Packages for Enterprise Linux 33 MB/s | 23 MB 00:00 2024-09-21T09:12:16.858 INFO:teuthology.orchestra.run.smithi078.stdout:lab-extras 38 kB/s | 1.7 kB 00:00 2024-09-21T09:12:18.748 INFO:teuthology.orchestra.run.smithi078.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-21T09:12:18.748 INFO:teuthology.orchestra.run.smithi078.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-21T09:12:18.843 INFO:teuthology.orchestra.run.smithi078.stdout:Dependencies resolved. 2024-09-21T09:12:18.853 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-21T09:12:18.853 INFO:teuthology.orchestra.run.smithi078.stdout: Package Architecture Version Repository Size 2024-09-21T09:12:18.854 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-21T09:12:18.854 INFO:teuthology.orchestra.run.smithi078.stdout:Installing: 2024-09-21T09:12:18.854 INFO:teuthology.orchestra.run.smithi078.stdout: kernel x86_64 5.14.0-508.el9 baseos 36 k 2024-09-21T09:12:18.854 INFO:teuthology.orchestra.run.smithi078.stdout:Installing dependencies: 2024-09-21T09:12:18.854 INFO:teuthology.orchestra.run.smithi078.stdout: kernel-core x86_64 5.14.0-508.el9 baseos 16 M 2024-09-21T09:12:18.854 INFO:teuthology.orchestra.run.smithi078.stdout: kernel-modules x86_64 5.14.0-508.el9 baseos 35 M 2024-09-21T09:12:18.854 INFO:teuthology.orchestra.run.smithi078.stdout: kernel-modules-core x86_64 5.14.0-508.el9 baseos 29 M 2024-09-21T09:12:18.854 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:12:18.854 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction Summary 2024-09-21T09:12:18.854 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-21T09:12:18.855 INFO:teuthology.orchestra.run.smithi078.stdout:Install 4 Packages 2024-09-21T09:12:18.855 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:12:18.855 INFO:teuthology.orchestra.run.smithi078.stdout:Total download size: 79 M 2024-09-21T09:12:18.855 INFO:teuthology.orchestra.run.smithi078.stdout:Installed size: 126 M 2024-09-21T09:12:18.855 INFO:teuthology.orchestra.run.smithi078.stdout:Downloading Packages: 2024-09-21T09:12:19.076 INFO:teuthology.orchestra.run.smithi078.stdout:(1/4): kernel-5.14.0-508.el9.x86_64.rpm 413 kB/s | 36 kB 00:00 2024-09-21T09:12:19.610 INFO:teuthology.orchestra.run.smithi078.stdout:(2/4): kernel-core-5.14.0-508.el9.x86_64.rpm 25 MB/s | 16 MB 00:00 2024-09-21T09:12:19.927 INFO:teuthology.orchestra.run.smithi078.stdout:(3/4): kernel-modules-5.14.0-508.el9.x86_64.rpm 37 MB/s | 35 MB 00:00 2024-09-21T09:12:20.253 INFO:teuthology.orchestra.run.smithi078.stdout:(4/4): kernel-modules-core-5.14.0-508.el9.x86_6 24 MB/s | 29 MB 00:01 2024-09-21T09:12:20.253 INFO:teuthology.orchestra.run.smithi078.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:12:20.253 INFO:teuthology.orchestra.run.smithi078.stdout:Total 56 MB/s | 79 MB 00:01 2024-09-21T09:12:20.938 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction check 2024-09-21T09:12:21.290 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction check succeeded. 2024-09-21T09:12:21.290 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction test 2024-09-21T09:12:22.254 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction test succeeded. 2024-09-21T09:12:22.255 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction 2024-09-21T09:12:24.676 INFO:teuthology.orchestra.run.smithi078.stdout: Preparing : 1/1 2024-09-21T09:12:25.422 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : kernel-modules-core-5.14.0-508.el9.x86_64 1/4 2024-09-21T09:12:25.520 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-21T09:12:28.273 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-21T09:12:28.399 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-21T09:12:33.234 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-21T09:12:33.274 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : kernel-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:12:38.114 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:13:16.137 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:13:16.177 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:13:16.545 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: kernel-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:13:16.545 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : kernel-5.14.0-508.el9.x86_64 1/4 2024-09-21T09:13:16.545 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-21T09:13:16.545 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-21T09:13:17.396 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:13:17.396 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:13:17.396 INFO:teuthology.orchestra.run.smithi078.stdout:Installed: 2024-09-21T09:13:17.396 INFO:teuthology.orchestra.run.smithi078.stdout: kernel-5.14.0-508.el9.x86_64 2024-09-21T09:13:17.396 INFO:teuthology.orchestra.run.smithi078.stdout: kernel-core-5.14.0-508.el9.x86_64 2024-09-21T09:13:17.397 INFO:teuthology.orchestra.run.smithi078.stdout: kernel-modules-5.14.0-508.el9.x86_64 2024-09-21T09:13:17.397 INFO:teuthology.orchestra.run.smithi078.stdout: kernel-modules-core-5.14.0-508.el9.x86_64 2024-09-21T09:13:17.397 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:13:17.397 INFO:teuthology.orchestra.run.smithi078.stdout:Complete! 2024-09-21T09:13:17.752 DEBUG:teuthology.orchestra.run.smithi078:> rpm -q kernel | sort -rV | head -n 1 2024-09-21T09:13:17.831 INFO:teuthology.orchestra.run.smithi078.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-21T09:13:17.831 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-21T09:13:17.831 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-508.el9.x86_64 2024-09-21T09:13:17.831 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-21T09:13:17.831 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-21T09:13:17.831 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2024-09-21T09:13:17.831 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi078.front.sepia.ceph.com, path=None, version=distro) 2024-09-21T09:13:17.832 DEBUG:teuthology.orchestra.run.smithi078:> rpm -q kernel | sort -rV | head -n 1 2024-09-21T09:13:17.889 INFO:teuthology.orchestra.run.smithi078.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-21T09:13:17.890 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-21T09:13:17.890 DEBUG:teuthology.orchestra.run.smithi078:> sudo rpm -qi grub2-tools 2024-09-21T09:13:17.927 INFO:teuthology.orchestra.run.smithi078.stdout:Name : grub2-tools 2024-09-21T09:13:17.927 INFO:teuthology.orchestra.run.smithi078.stdout:Epoch : 1 2024-09-21T09:13:17.927 INFO:teuthology.orchestra.run.smithi078.stdout:Version : 2.06 2024-09-21T09:13:17.927 INFO:teuthology.orchestra.run.smithi078.stdout:Release : 82.el9 2024-09-21T09:13:17.927 INFO:teuthology.orchestra.run.smithi078.stdout:Architecture: x86_64 2024-09-21T09:13:17.927 INFO:teuthology.orchestra.run.smithi078.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-21T09:13:17.927 INFO:teuthology.orchestra.run.smithi078.stdout:Group : Unspecified 2024-09-21T09:13:17.927 INFO:teuthology.orchestra.run.smithi078.stdout:Size : 8274670 2024-09-21T09:13:17.928 INFO:teuthology.orchestra.run.smithi078.stdout:License : GPLv3+ 2024-09-21T09:13:17.928 INFO:teuthology.orchestra.run.smithi078.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-21T09:13:17.928 INFO:teuthology.orchestra.run.smithi078.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-21T09:13:17.928 INFO:teuthology.orchestra.run.smithi078.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-21T09:13:17.928 INFO:teuthology.orchestra.run.smithi078.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-21T09:13:17.928 INFO:teuthology.orchestra.run.smithi078.stdout:Packager : builder@centos.org 2024-09-21T09:13:17.928 INFO:teuthology.orchestra.run.smithi078.stdout:Vendor : CentOS 2024-09-21T09:13:17.928 INFO:teuthology.orchestra.run.smithi078.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-21T09:13:17.928 INFO:teuthology.orchestra.run.smithi078.stdout:Summary : Support tools for GRUB. 2024-09-21T09:13:17.928 INFO:teuthology.orchestra.run.smithi078.stdout:Description : 2024-09-21T09:13:17.928 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:13:17.928 INFO:teuthology.orchestra.run.smithi078.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-21T09:13:17.928 INFO:teuthology.orchestra.run.smithi078.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-21T09:13:17.928 INFO:teuthology.orchestra.run.smithi078.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-21T09:13:17.929 INFO:teuthology.orchestra.run.smithi078.stdout:hardware devices. 2024-09-21T09:13:17.929 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:13:17.929 INFO:teuthology.orchestra.run.smithi078.stdout:This subpackage provides tools for support of all platforms. 2024-09-21T09:13:17.929 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-21T09:13:17.930 INFO:teuthology.task.kernel:Updating grub on smithi078 to boot 5.14.0-508.el9.x86_64 2024-09-21T09:13:17.930 DEBUG:teuthology.orchestra.run.smithi078:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-21T09:13:18.650 INFO:teuthology.orchestra.run.smithi078.stderr:Generating grub configuration file ... 2024-09-21T09:13:19.968 INFO:teuthology.orchestra.run.smithi078.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-21T09:13:20.009 INFO:teuthology.orchestra.run.smithi078.stderr:done 2024-09-21T09:13:20.011 DEBUG:teuthology.orchestra.run.smithi078:> mktemp 2024-09-21T09:13:20.026 INFO:teuthology.orchestra.run.smithi078.stdout:/tmp/tmp.DRYBoMAujx 2024-09-21T09:13:20.026 DEBUG:teuthology.orchestra.run.smithi078:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.DRYBoMAujx 2024-09-21T09:13:20.093 DEBUG:teuthology.orchestra.run.smithi078:> sudo chmod 0666 /tmp/tmp.DRYBoMAujx 2024-09-21T09:13:20.237 DEBUG:teuthology.orchestra.remote:smithi078:/tmp/tmp.DRYBoMAujx is 6KB 2024-09-21T09:13:20.258 DEBUG:teuthology.orchestra.run.smithi078:> rm -fr /tmp/tmp.DRYBoMAujx 2024-09-21T09:13:20.273 DEBUG:teuthology.orchestra.run.smithi078:> sudo /bin/ls /boot/loader/entries || true 2024-09-21T09:13:20.340 INFO:teuthology.orchestra.run.smithi078.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-21T09:13:20.341 INFO:teuthology.orchestra.run.smithi078.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-21T09:13:20.341 INFO:teuthology.orchestra.run.smithi078.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-21T09:13:20.341 INFO:teuthology.orchestra.run.smithi078.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64.conf 2024-09-21T09:13:20.342 DEBUG:teuthology.orchestra.run.smithi078:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64 2024-09-21T09:13:20.491 DEBUG:teuthology.orchestra.run.smithi078:> sudo shutdown -r now 2024-09-21T09:13:50.522 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2024-09-21T09:13:50.523 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi078.front.sepia.ceph.com' 2024-09-21T09:13:50.524 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi078.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T09:14:09.007 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.78 2024-09-21T09:14:18.013 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi078.front.sepia.ceph.com' 2024-09-21T09:14:18.014 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi078.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T09:14:21.071 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.78 2024-09-21T09:14:33.082 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi078.front.sepia.ceph.com' 2024-09-21T09:14:33.083 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi078.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T09:15:04.942 DEBUG:teuthology.orchestra.run.smithi078:> true 2024-09-21T09:15:05.485 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi078.front.sepia.ceph.com' 2024-09-21T09:15:05.485 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "5.14.0-508.el9.x86_64"... 2024-09-21T09:15:05.485 DEBUG:teuthology.orchestra.run.smithi078:> uname -r 2024-09-21T09:15:05.514 INFO:teuthology.orchestra.run.smithi078.stdout:5.14.0-508.el9.x86_64 2024-09-21T09:15:05.514 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-508.el9.x86_64 vs 5.14.0-508.el9.x86_64 2024-09-21T09:15:05.514 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-21T09:15:05.514 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-21T09:15:06.516 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-09-21T09:15:06.516 DEBUG:teuthology.orchestra.run.smithi078:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-21T09:15:06.729 INFO:teuthology.orchestra.run.smithi078.stdout:ttyS1 2024-09-21T09:15:06.964 DEBUG:teuthology.parallel:result is None 2024-09-21T09:15:06.964 INFO:teuthology.run_tasks:Running task internal.base... 2024-09-21T09:15:06.970 INFO:teuthology.task.internal:Creating test directory... 2024-09-21T09:15:06.970 DEBUG:teuthology.orchestra.run.smithi078:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-21T09:15:06.986 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-09-21T09:15:06.992 INFO:teuthology.run_tasks:Running task internal.archive... 2024-09-21T09:15:06.998 INFO:teuthology.task.internal:Creating archive directory... 2024-09-21T09:15:06.999 DEBUG:teuthology.orchestra.run.smithi078:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-21T09:15:07.048 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-09-21T09:15:07.054 INFO:teuthology.task.internal:Enabling coredump saving... 2024-09-21T09:15:07.055 DEBUG:teuthology.orchestra.run.smithi078:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-21T09:15:07.118 INFO:teuthology.orchestra.run.smithi078.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-21T09:15:07.128 INFO:teuthology.orchestra.run.smithi078.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-21T09:15:07.130 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-09-21T09:15:07.137 INFO:teuthology.task.internal:Configuring sudo... 2024-09-21T09:15:07.137 DEBUG:teuthology.orchestra.run.smithi078:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-21T09:15:07.195 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-09-21T09:15:07.204 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-09-21T09:15:07.204 DEBUG:teuthology.orchestra.run.smithi078:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-21T09:15:07.250 DEBUG:teuthology.orchestra.run.smithi078:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-21T09:15:07.333 DEBUG:teuthology.orchestra.run.smithi078:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-21T09:15:07.378 DEBUG:teuthology.orchestra.run.smithi078:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-21T09:15:07.460 DEBUG:teuthology.orchestra.run.smithi078:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-21T09:15:07.499 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-21T09:15:07.499 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-21T09:15:07.563 DEBUG:teuthology.orchestra.run.smithi078:> sudo service rsyslog restart 2024-09-21T09:15:07.649 INFO:teuthology.orchestra.run.smithi078.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-21T09:15:08.159 INFO:teuthology.run_tasks:Running task internal.timer... 2024-09-21T09:15:08.168 INFO:teuthology.task.internal:Starting timer... 2024-09-21T09:15:08.168 INFO:teuthology.run_tasks:Running task pcp... 2024-09-21T09:15:08.231 INFO:teuthology.run_tasks:Running task selinux... 2024-09-21T09:15:08.244 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0']} 2024-09-21T09:15:08.244 DEBUG:teuthology.orchestra.run.smithi078:> sudo service auditd rotate 2024-09-21T09:15:08.302 INFO:teuthology.orchestra.run.smithi078.stdout:Rotating logs: 2024-09-21T09:15:08.304 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-09-21T09:15:08.304 DEBUG:teuthology.orchestra.run.smithi078:> /usr/sbin/getenforce 2024-09-21T09:15:08.368 INFO:teuthology.orchestra.run.smithi078.stdout:Permissive 2024-09-21T09:15:08.368 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi078.front.sepia.ceph.com': 'permissive'} 2024-09-21T09:15:08.368 DEBUG:teuthology.orchestra.run.smithi078:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-21T09:15:08.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:15:08.394 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-09-21T09:15:08.394 DEBUG:teuthology.orchestra.run.smithi078:> sudo /usr/sbin/setenforce permissive 2024-09-21T09:15:08.465 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-09-21T09:15:08.482 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2024-09-21T09:15:10.639 INFO:teuthology.repo_utils:Fetching git.ceph.com_ceph-cm-ansible_main from origin 2024-09-21T09:15:10.848 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-09-21T09:15:10.892 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-09-21T09:15:10.893 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi078.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-09-21T09:18:50.314 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi078.front.sepia.ceph.com')] 2024-09-21T09:18:50.315 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi078.front.sepia.ceph.com' 2024-09-21T09:18:50.316 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi078.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T09:18:50.393 DEBUG:teuthology.orchestra.run.smithi078:> true 2024-09-21T09:18:50.475 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi078.front.sepia.ceph.com' 2024-09-21T09:18:50.475 INFO:teuthology.run_tasks:Running task clock... 2024-09-21T09:18:50.484 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-09-21T09:18:50.485 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-21T09:18:50.485 DEBUG:teuthology.orchestra.run.smithi078:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-21T09:18:50.549 INFO:teuthology.orchestra.run.smithi078.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-21T09:18:50.566 INFO:teuthology.orchestra.run.smithi078.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-21T09:18:50.595 INFO:teuthology.orchestra.run.smithi078.stderr:sudo: ntpd: command not found 2024-09-21T09:18:50.608 INFO:teuthology.orchestra.run.smithi078.stdout:506 Cannot talk to daemon 2024-09-21T09:18:50.625 INFO:teuthology.orchestra.run.smithi078.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-21T09:18:50.642 INFO:teuthology.orchestra.run.smithi078.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-21T09:18:50.708 INFO:teuthology.orchestra.run.smithi078.stderr:bash: line 1: ntpq: command not found 2024-09-21T09:18:50.712 INFO:teuthology.orchestra.run.smithi078.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-21T09:18:50.712 INFO:teuthology.orchestra.run.smithi078.stdout:=============================================================================== 2024-09-21T09:18:50.712 INFO:teuthology.orchestra.run.smithi078.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T09:18:50.712 INFO:teuthology.orchestra.run.smithi078.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T09:18:50.712 INFO:teuthology.orchestra.run.smithi078.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T09:18:50.713 INFO:teuthology.orchestra.run.smithi078.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T09:18:50.713 INFO:teuthology.run_tasks:Running task install... 2024-09-21T09:18:50.722 DEBUG:teuthology.task.install:project ceph 2024-09-21T09:18:50.722 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208'}} 2024-09-21T09:18:50.723 DEBUG:teuthology.task.install:config {'branch': 'reef', 'exclude_packages': ['librados3', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'cephadm', 'ceph-volume'], 'extra_packages': ['librados2'], 'flavor': 'default', 'sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208'} 2024-09-21T09:18:50.723 INFO:teuthology.task.install:Using flavor: default 2024-09-21T09:18:50.730 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2024-09-21T09:18:50.730 INFO:teuthology.task.install:extra packages: ['librados2'] 2024-09-21T09:18:50.731 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'reef', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['librados3', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'cephadm', 'ceph-volume'], 'extra_packages': ['librados2'], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208', 'tag': None, 'wait_for_package': False} 2024-09-21T09:18:50.731 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-21T09:18:50.731 INFO:teuthology.packaging:ref: None 2024-09-21T09:18:50.731 INFO:teuthology.packaging:tag: None 2024-09-21T09:18:50.732 INFO:teuthology.packaging:branch: reef 2024-09-21T09:18:50.732 INFO:teuthology.packaging:sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:18:50.732 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=reef 2024-09-21T09:18:50.985 INFO:teuthology.task.install.rpm:Pulling from https://4.chacra.ceph.com/r/ceph/reef/6a1bdf854bb5803efd58957907442aa01f41a8fc/centos/9/flavors/default/ 2024-09-21T09:18:50.986 INFO:teuthology.task.install.rpm:Package version is 18.2.4-871.g6a1bdf85 2024-09-21T09:18:51.093 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://4.chacra.ceph.com/r/ceph/reef/6a1bdf854bb5803efd58957907442aa01f41a8fc/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://4.chacra.ceph.com/r/ceph/reef/6a1bdf854bb5803efd58957907442aa01f41a8fc/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://4.chacra.ceph.com/r/ceph/reef/6a1bdf854bb5803efd58957907442aa01f41a8fc/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-09-21T09:18:51.093 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-21T09:18:51.093 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-09-21T09:18:51.123 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, ceph-immutable-object-cache, ceph-mgr, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd, librados2 on remote rpm x86_64 2024-09-21T09:18:51.123 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-21T09:18:51.123 INFO:teuthology.packaging:ref: None 2024-09-21T09:18:51.123 INFO:teuthology.packaging:tag: None 2024-09-21T09:18:51.123 INFO:teuthology.packaging:branch: reef 2024-09-21T09:18:51.123 INFO:teuthology.packaging:sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:18:51.124 DEBUG:teuthology.orchestra.run.smithi078:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/reef/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-09-21T09:18:51.194 DEBUG:teuthology.orchestra.run.smithi078:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-09-21T09:18:51.276 DEBUG:teuthology.orchestra.run.smithi078:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-09-21T09:18:51.346 INFO:teuthology.orchestra.run.smithi078.stdout:check_obsoletes = 1 2024-09-21T09:18:51.348 DEBUG:teuthology.orchestra.run.smithi078:> sudo yum clean all 2024-09-21T09:18:51.587 INFO:teuthology.orchestra.run.smithi078.stdout:45 files removed 2024-09-21T09:18:51.614 DEBUG:teuthology.orchestra.run.smithi078:> sudo yum -y install ceph-radosgw 2024-09-21T09:18:52.177 INFO:teuthology.orchestra.run.smithi078.stdout:ceph packages for x86_64 274 kB/s | 77 kB 00:00 2024-09-21T09:18:52.453 INFO:teuthology.orchestra.run.smithi078.stdout:ceph noarch packages 49 kB/s | 11 kB 00:00 2024-09-21T09:18:52.720 INFO:teuthology.orchestra.run.smithi078.stdout:ceph source packages 8.2 kB/s | 1.9 kB 00:00 2024-09-21T09:18:53.387 INFO:teuthology.orchestra.run.smithi078.stdout:CentOS Stream 9 - BaseOS 13 MB/s | 8.3 MB 00:00 2024-09-21T09:18:55.013 INFO:teuthology.orchestra.run.smithi078.stdout:CentOS Stream 9 - AppStream 35 MB/s | 20 MB 00:00 2024-09-21T09:18:59.231 INFO:teuthology.orchestra.run.smithi078.stdout:CentOS Stream 9 - CRB 14 MB/s | 6.5 MB 00:00 2024-09-21T09:19:00.698 INFO:teuthology.orchestra.run.smithi078.stdout:CentOS Stream 9 - Extras packages 77 kB/s | 19 kB 00:00 2024-09-21T09:19:01.511 INFO:teuthology.orchestra.run.smithi078.stdout:Extra Packages for Enterprise Linux 33 MB/s | 23 MB 00:00 2024-09-21T09:19:06.917 INFO:teuthology.orchestra.run.smithi078.stdout:lab-extras 32 kB/s | 1.7 kB 00:00 2024-09-21T09:19:09.100 INFO:teuthology.orchestra.run.smithi078.stdout:Dependencies resolved. 2024-09-21T09:19:09.133 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-21T09:19:09.133 INFO:teuthology.orchestra.run.smithi078.stdout: Package Arch Version Repository Size 2024-09-21T09:19:09.133 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-21T09:19:09.133 INFO:teuthology.orchestra.run.smithi078.stdout:Installing: 2024-09-21T09:19:09.133 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-radosgw x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 7.7 M 2024-09-21T09:19:09.133 INFO:teuthology.orchestra.run.smithi078.stdout:Upgrading: 2024-09-21T09:19:09.134 INFO:teuthology.orchestra.run.smithi078.stdout: librados2 x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 3.3 M 2024-09-21T09:19:09.134 INFO:teuthology.orchestra.run.smithi078.stdout: librbd1 x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 3.0 M 2024-09-21T09:19:09.134 INFO:teuthology.orchestra.run.smithi078.stdout:Installing dependencies: 2024-09-21T09:19:09.134 INFO:teuthology.orchestra.run.smithi078.stdout: boost-program-options x86_64 1.75.0-8.el9 appstream 107 k 2024-09-21T09:19:09.134 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-base x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 5.1 M 2024-09-21T09:19:09.134 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-common x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 18 M 2024-09-21T09:19:09.134 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-selinux x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 25 k 2024-09-21T09:19:09.134 INFO:teuthology.orchestra.run.smithi078.stdout: fmt x86_64 8.1.1-5.el9 epel 111 k 2024-09-21T09:19:09.134 INFO:teuthology.orchestra.run.smithi078.stdout: gperftools-libs x86_64 2.9.1-3.el9 epel 308 k 2024-09-21T09:19:09.134 INFO:teuthology.orchestra.run.smithi078.stdout: libarrow x86_64 9.0.0-11.el9 epel 4.4 M 2024-09-21T09:19:09.134 INFO:teuthology.orchestra.run.smithi078.stdout: libarrow-doc noarch 9.0.0-11.el9 epel 26 k 2024-09-21T09:19:09.134 INFO:teuthology.orchestra.run.smithi078.stdout: libcephfs2 x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 711 k 2024-09-21T09:19:09.135 INFO:teuthology.orchestra.run.smithi078.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-09-21T09:19:09.135 INFO:teuthology.orchestra.run.smithi078.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2024-09-21T09:19:09.135 INFO:teuthology.orchestra.run.smithi078.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2024-09-21T09:19:09.135 INFO:teuthology.orchestra.run.smithi078.stdout: libradosstriper1 x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 477 k 2024-09-21T09:19:09.135 INFO:teuthology.orchestra.run.smithi078.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2024-09-21T09:19:09.135 INFO:teuthology.orchestra.run.smithi078.stdout: librgw2 x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 4.5 M 2024-09-21T09:19:09.135 INFO:teuthology.orchestra.run.smithi078.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-09-21T09:19:09.135 INFO:teuthology.orchestra.run.smithi078.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2024-09-21T09:19:09.135 INFO:teuthology.orchestra.run.smithi078.stdout: parquet-libs x86_64 9.0.0-11.el9 epel 839 k 2024-09-21T09:19:09.135 INFO:teuthology.orchestra.run.smithi078.stdout: python3-ceph-argparse x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 45 k 2024-09-21T09:19:09.135 INFO:teuthology.orchestra.run.smithi078.stdout: python3-ceph-common x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 129 k 2024-09-21T09:19:09.135 INFO:teuthology.orchestra.run.smithi078.stdout: python3-cephfs x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 161 k 2024-09-21T09:19:09.136 INFO:teuthology.orchestra.run.smithi078.stdout: python3-prettytable noarch 0.7.2-27.el9 appstream 42 k 2024-09-21T09:19:09.136 INFO:teuthology.orchestra.run.smithi078.stdout: python3-rados x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 320 k 2024-09-21T09:19:09.136 INFO:teuthology.orchestra.run.smithi078.stdout: python3-rbd x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 300 k 2024-09-21T09:19:09.136 INFO:teuthology.orchestra.run.smithi078.stdout: python3-rgw x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 99 k 2024-09-21T09:19:09.136 INFO:teuthology.orchestra.run.smithi078.stdout: re2 x86_64 1:20211101-20.el9 epel 191 k 2024-09-21T09:19:09.136 INFO:teuthology.orchestra.run.smithi078.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-09-21T09:19:09.136 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:19:09.136 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction Summary 2024-09-21T09:19:09.136 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-21T09:19:09.136 INFO:teuthology.orchestra.run.smithi078.stdout:Install 28 Packages 2024-09-21T09:19:09.136 INFO:teuthology.orchestra.run.smithi078.stdout:Upgrade 2 Packages 2024-09-21T09:19:09.136 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:19:09.137 INFO:teuthology.orchestra.run.smithi078.stdout:Total download size: 53 M 2024-09-21T09:19:09.137 INFO:teuthology.orchestra.run.smithi078.stdout:Downloading Packages: 2024-09-21T09:19:09.785 INFO:teuthology.orchestra.run.smithi078.stdout:(1/30): ceph-base-18.2.4-871.g6a1bdf85.el9.x86_ 11 MB/s | 5.1 MB 00:00 2024-09-21T09:19:09.843 INFO:teuthology.orchestra.run.smithi078.stdout:(2/30): ceph-selinux-18.2.4-871.g6a1bdf85.el9.x 432 kB/s | 25 kB 00:00 2024-09-21T09:19:09.927 INFO:teuthology.orchestra.run.smithi078.stdout:(3/30): ceph-radosgw-18.2.4-871.g6a1bdf85.el9.x 13 MB/s | 7.7 MB 00:00 2024-09-21T09:19:09.969 INFO:teuthology.orchestra.run.smithi078.stdout:(4/30): libcephfs2-18.2.4-871.g6a1bdf85.el9.x86 5.6 MB/s | 711 kB 00:00 2024-09-21T09:19:09.994 INFO:teuthology.orchestra.run.smithi078.stdout:(5/30): libradosstriper1-18.2.4-871.g6a1bdf85.e 7.0 MB/s | 477 kB 00:00 2024-09-21T09:19:10.061 INFO:teuthology.orchestra.run.smithi078.stdout:(6/30): python3-ceph-argparse-18.2.4-871.g6a1bd 673 kB/s | 45 kB 00:00 2024-09-21T09:19:10.178 INFO:teuthology.orchestra.run.smithi078.stdout:(7/30): librgw2-18.2.4-871.g6a1bdf85.el9.x86_64 21 MB/s | 4.5 MB 00:00 2024-09-21T09:19:10.211 INFO:teuthology.orchestra.run.smithi078.stdout:(8/30): python3-ceph-common-18.2.4-871.g6a1bdf8 861 kB/s | 129 kB 00:00 2024-09-21T09:19:10.245 INFO:teuthology.orchestra.run.smithi078.stdout:(9/30): python3-cephfs-18.2.4-871.g6a1bdf85.el9 2.4 MB/s | 161 kB 00:00 2024-09-21T09:19:10.270 INFO:teuthology.orchestra.run.smithi078.stdout:(10/30): python3-rados-18.2.4-871.g6a1bdf85.el9 5.3 MB/s | 320 kB 00:00 2024-09-21T09:19:10.295 INFO:teuthology.orchestra.run.smithi078.stdout:(11/30): python3-rbd-18.2.4-871.g6a1bdf85.el9.x 5.9 MB/s | 300 kB 00:00 2024-09-21T09:19:10.321 INFO:teuthology.orchestra.run.smithi078.stdout:(12/30): python3-rgw-18.2.4-871.g6a1bdf85.el9.x 1.9 MB/s | 99 kB 00:00 2024-09-21T09:19:10.555 INFO:teuthology.orchestra.run.smithi078.stdout:(13/30): ceph-common-18.2.4-871.g6a1bdf85.el9.x 15 MB/s | 18 MB 00:01 2024-09-21T09:19:10.613 INFO:teuthology.orchestra.run.smithi078.stdout:(14/30): boost-program-options-1.75.0-8.el9.x86 337 kB/s | 107 kB 00:00 2024-09-21T09:19:10.663 INFO:teuthology.orchestra.run.smithi078.stdout:(15/30): libpmemobj-1.12.1-1.el9.x86_64.rpm 468 kB/s | 160 kB 00:00 2024-09-21T09:19:10.688 INFO:teuthology.orchestra.run.smithi078.stdout:(16/30): librabbitmq-0.11.0-7.el9.x86_64.rpm 339 kB/s | 45 kB 00:00 2024-09-21T09:19:10.764 INFO:teuthology.orchestra.run.smithi078.stdout:(17/30): python3-prettytable-0.7.2-27.el9.noarc 563 kB/s | 42 kB 00:00 2024-09-21T09:19:10.789 INFO:teuthology.orchestra.run.smithi078.stdout:(18/30): lttng-ust-2.12.0-6.el9.x86_64.rpm 2.3 MB/s | 292 kB 00:00 2024-09-21T09:19:10.840 INFO:teuthology.orchestra.run.smithi078.stdout:(19/30): librdkafka-1.6.1-102.el9.x86_64.rpm 2.9 MB/s | 662 kB 00:00 2024-09-21T09:19:10.874 INFO:teuthology.orchestra.run.smithi078.stdout:(20/30): fmt-8.1.1-5.el9.x86_64.rpm 1.0 MB/s | 111 kB 00:00 2024-09-21T09:19:10.898 INFO:teuthology.orchestra.run.smithi078.stdout:(21/30): libarrow-doc-9.0.0-11.el9.noarch.rpm 1.1 MB/s | 26 kB 00:00 2024-09-21T09:19:10.924 INFO:teuthology.orchestra.run.smithi078.stdout:(22/30): liboath-2.6.7-2.el9.x86_64.rpm 1.9 MB/s | 49 kB 00:00 2024-09-21T09:19:10.957 INFO:teuthology.orchestra.run.smithi078.stdout:(23/30): gperftools-libs-2.9.1-3.el9.x86_64.rpm 1.8 MB/s | 308 kB 00:00 2024-09-21T09:19:10.983 INFO:teuthology.orchestra.run.smithi078.stdout:(24/30): libunwind-1.6.2-1.el9.x86_64.rpm 1.1 MB/s | 67 kB 00:00 2024-09-21T09:19:11.016 INFO:teuthology.orchestra.run.smithi078.stdout:(25/30): re2-20211101-20.el9.x86_64.rpm 5.6 MB/s | 191 kB 00:00 2024-09-21T09:19:11.058 INFO:teuthology.orchestra.run.smithi078.stdout:(26/30): parquet-libs-9.0.0-11.el9.x86_64.rpm 8.2 MB/s | 839 kB 00:00 2024-09-21T09:19:11.117 INFO:teuthology.orchestra.run.smithi078.stdout:(27/30): thrift-0.15.0-2.el9.x86_64.rpm 16 MB/s | 1.6 MB 00:00 2024-09-21T09:19:11.251 INFO:teuthology.orchestra.run.smithi078.stdout:(28/30): libarrow-9.0.0-11.el9.x86_64.rpm 11 MB/s | 4.4 MB 00:00 2024-09-21T09:19:11.309 INFO:teuthology.orchestra.run.smithi078.stdout:(29/30): librados2-18.2.4-871.g6a1bdf85.el9.x86 13 MB/s | 3.3 MB 00:00 2024-09-21T09:19:11.384 INFO:teuthology.orchestra.run.smithi078.stdout:(30/30): librbd1-18.2.4-871.g6a1bdf85.el9.x86_6 11 MB/s | 3.0 MB 00:00 2024-09-21T09:19:11.389 INFO:teuthology.orchestra.run.smithi078.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:19:11.389 INFO:teuthology.orchestra.run.smithi078.stdout:Total 24 MB/s | 53 MB 00:02 2024-09-21T09:19:11.458 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction check 2024-09-21T09:19:11.492 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction check succeeded. 2024-09-21T09:19:11.492 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction test 2024-09-21T09:19:12.121 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction test succeeded. 2024-09-21T09:19:12.122 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction 2024-09-21T09:19:13.568 INFO:teuthology.orchestra.run.smithi078.stdout: Preparing : 1/1 2024-09-21T09:19:13.649 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/32 2024-09-21T09:19:13.689 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 2/32 2024-09-21T09:19:13.747 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 3/32 2024-09-21T09:19:14.123 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : thrift-0.15.0-2.el9.x86_64 4/32 2024-09-21T09:19:14.180 INFO:teuthology.orchestra.run.smithi078.stdout: Upgrading : librados2-2:18.2.4-871.g6a1bdf85.el9.x86_64 5/32 2024-09-21T09:19:14.255 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: librados2-2:18.2.4-871.g6a1bdf85.el9.x86_64 5/32 2024-09-21T09:19:14.348 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-rados-2:18.2.4-871.g6a1bdf85.el9.x86_64 6/32 2024-09-21T09:19:14.390 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : libcephfs2-2:18.2.4-871.g6a1bdf85.el9.x86_64 7/32 2024-09-21T09:19:14.453 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: libcephfs2-2:18.2.4-871.g6a1bdf85.el9.x86_64 7/32 2024-09-21T09:19:14.489 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : liboath-2.6.7-2.el9.x86_64 8/32 2024-09-21T09:19:14.533 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-ceph-argparse-2:18.2.4-871.g6a1bdf85.el9.x 9/32 2024-09-21T09:19:14.599 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-cephfs-2:18.2.4-871.g6a1bdf85.el9.x86_64 10/32 2024-09-21T09:19:14.625 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : libradosstriper1-2:18.2.4-871.g6a1bdf85.el9.x86_64 11/32 2024-09-21T09:19:14.691 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: libradosstriper1-2:18.2.4-871.g6a1bdf85.el9.x86_64 11/32 2024-09-21T09:19:14.795 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : re2-1:20211101-20.el9.x86_64 12/32 2024-09-21T09:19:14.824 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : libarrow-9.0.0-11.el9.x86_64 13/32 2024-09-21T09:19:14.898 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 14/32 2024-09-21T09:19:14.939 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : gperftools-libs-2.9.1-3.el9.x86_64 15/32 2024-09-21T09:19:14.966 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : libarrow-doc-9.0.0-11.el9.noarch 16/32 2024-09-21T09:19:15.019 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : fmt-8.1.1-5.el9.x86_64 17/32 2024-09-21T09:19:15.054 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 18/32 2024-09-21T09:19:15.369 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 19/32 2024-09-21T09:19:15.415 INFO:teuthology.orchestra.run.smithi078.stdout: Upgrading : librbd1-2:18.2.4-871.g6a1bdf85.el9.x86_64 20/32 2024-09-21T09:19:15.505 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: librbd1-2:18.2.4-871.g6a1bdf85.el9.x86_64 20/32 2024-09-21T09:19:15.538 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-rbd-2:18.2.4-871.g6a1bdf85.el9.x86_64 21/32 2024-09-21T09:19:15.571 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : boost-program-options-1.75.0-8.el9.x86_64 22/32 2024-09-21T09:19:15.975 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : parquet-libs-9.0.0-11.el9.x86_64 23/32 2024-09-21T09:19:16.001 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : librgw2-2:18.2.4-871.g6a1bdf85.el9.x86_64 24/32 2024-09-21T09:19:16.074 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: librgw2-2:18.2.4-871.g6a1bdf85.el9.x86_64 24/32 2024-09-21T09:19:16.150 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-rgw-2:18.2.4-871.g6a1bdf85.el9.x86_64 25/32 2024-09-21T09:19:16.186 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-ceph-common-2:18.2.4-871.g6a1bdf85.el9.x86 26/32 2024-09-21T09:19:18.317 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: ceph-common-2:18.2.4-871.g6a1bdf85.el9.x86_64 27/32 2024-09-21T09:19:18.349 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : ceph-common-2:18.2.4-871.g6a1bdf85.el9.x86_64 27/32 2024-09-21T09:19:18.886 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: ceph-common-2:18.2.4-871.g6a1bdf85.el9.x86_64 27/32 2024-09-21T09:19:18.927 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : ceph-base-2:18.2.4-871.g6a1bdf85.el9.x86_64 28/32 2024-09-21T09:19:19.028 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: ceph-base-2:18.2.4-871.g6a1bdf85.el9.x86_64 28/32 2024-09-21T09:19:19.028 INFO:teuthology.orchestra.run.smithi078.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-09-21T09:19:19.029 INFO:teuthology.orchestra.run.smithi078.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-09-21T09:19:19.029 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:19:19.070 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : ceph-selinux-2:18.2.4-871.g6a1bdf85.el9.x86_64 29/32 2024-09-21T09:19:29.034 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: ceph-selinux-2:18.2.4-871.g6a1bdf85.el9.x86_64 29/32 2024-09-21T09:19:29.035 INFO:teuthology.orchestra.run.smithi078.stdout:skipping the directory /sys 2024-09-21T09:19:29.035 INFO:teuthology.orchestra.run.smithi078.stdout:skipping the directory /proc 2024-09-21T09:19:29.035 INFO:teuthology.orchestra.run.smithi078.stdout:skipping the directory /mnt 2024-09-21T09:19:29.035 INFO:teuthology.orchestra.run.smithi078.stdout:skipping the directory /var/tmp 2024-09-21T09:19:29.035 INFO:teuthology.orchestra.run.smithi078.stdout:skipping the directory /home 2024-09-21T09:19:29.035 INFO:teuthology.orchestra.run.smithi078.stdout:skipping the directory /root 2024-09-21T09:19:29.035 INFO:teuthology.orchestra.run.smithi078.stdout:skipping the directory /tmp 2024-09-21T09:19:29.035 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:19:29.795 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : ceph-radosgw-2:18.2.4-871.g6a1bdf85.el9.x86_64 30/32 2024-09-21T09:19:29.820 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: ceph-radosgw-2:18.2.4-871.g6a1bdf85.el9.x86_64 30/32 2024-09-21T09:19:29.821 INFO:teuthology.orchestra.run.smithi078.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:19:29.821 INFO:teuthology.orchestra.run.smithi078.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-09-21T09:19:29.821 INFO:teuthology.orchestra.run.smithi078.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-09-21T09:19:29.821 INFO:teuthology.orchestra.run.smithi078.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-09-21T09:19:29.821 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:19:29.822 INFO:teuthology.orchestra.run.smithi078.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 31/32 2024-09-21T09:19:29.906 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 31/32 2024-09-21T09:19:29.906 INFO:teuthology.orchestra.run.smithi078.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 32/32 2024-09-21T09:19:31.146 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 32/32 2024-09-21T09:19:31.146 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : ceph-base-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/32 2024-09-21T09:19:31.146 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : ceph-common-2:18.2.4-871.g6a1bdf85.el9.x86_64 2/32 2024-09-21T09:19:31.146 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : ceph-radosgw-2:18.2.4-871.g6a1bdf85.el9.x86_64 3/32 2024-09-21T09:19:31.146 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : ceph-selinux-2:18.2.4-871.g6a1bdf85.el9.x86_64 4/32 2024-09-21T09:19:31.146 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : libcephfs2-2:18.2.4-871.g6a1bdf85.el9.x86_64 5/32 2024-09-21T09:19:31.146 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : libradosstriper1-2:18.2.4-871.g6a1bdf85.el9.x86_64 6/32 2024-09-21T09:19:31.146 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : librgw2-2:18.2.4-871.g6a1bdf85.el9.x86_64 7/32 2024-09-21T09:19:31.147 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-ceph-argparse-2:18.2.4-871.g6a1bdf85.el9.x 8/32 2024-09-21T09:19:31.147 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-ceph-common-2:18.2.4-871.g6a1bdf85.el9.x86 9/32 2024-09-21T09:19:31.147 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-cephfs-2:18.2.4-871.g6a1bdf85.el9.x86_64 10/32 2024-09-21T09:19:31.147 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-rados-2:18.2.4-871.g6a1bdf85.el9.x86_64 11/32 2024-09-21T09:19:31.147 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-rbd-2:18.2.4-871.g6a1bdf85.el9.x86_64 12/32 2024-09-21T09:19:31.147 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-rgw-2:18.2.4-871.g6a1bdf85.el9.x86_64 13/32 2024-09-21T09:19:31.147 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : boost-program-options-1.75.0-8.el9.x86_64 14/32 2024-09-21T09:19:31.147 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 15/32 2024-09-21T09:19:31.147 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 16/32 2024-09-21T09:19:31.147 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 17/32 2024-09-21T09:19:31.147 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 18/32 2024-09-21T09:19:31.147 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 19/32 2024-09-21T09:19:31.147 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : fmt-8.1.1-5.el9.x86_64 20/32 2024-09-21T09:19:31.147 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : gperftools-libs-2.9.1-3.el9.x86_64 21/32 2024-09-21T09:19:31.148 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : libarrow-9.0.0-11.el9.x86_64 22/32 2024-09-21T09:19:31.148 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : libarrow-doc-9.0.0-11.el9.noarch 23/32 2024-09-21T09:19:31.148 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 24/32 2024-09-21T09:19:31.148 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 25/32 2024-09-21T09:19:31.148 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : parquet-libs-9.0.0-11.el9.x86_64 26/32 2024-09-21T09:19:31.148 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : re2-1:20211101-20.el9.x86_64 27/32 2024-09-21T09:19:31.148 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 28/32 2024-09-21T09:19:31.149 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : librados2-2:18.2.4-871.g6a1bdf85.el9.x86_64 29/32 2024-09-21T09:19:31.149 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 30/32 2024-09-21T09:19:31.149 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : librbd1-2:18.2.4-871.g6a1bdf85.el9.x86_64 31/32 2024-09-21T09:19:32.122 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 32/32 2024-09-21T09:19:32.123 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:19:32.123 INFO:teuthology.orchestra.run.smithi078.stdout:Upgraded: 2024-09-21T09:19:32.123 INFO:teuthology.orchestra.run.smithi078.stdout: librados2-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:19:32.123 INFO:teuthology.orchestra.run.smithi078.stdout: librbd1-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:19:32.123 INFO:teuthology.orchestra.run.smithi078.stdout:Installed: 2024-09-21T09:19:32.123 INFO:teuthology.orchestra.run.smithi078.stdout: boost-program-options-1.75.0-8.el9.x86_64 2024-09-21T09:19:32.123 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-base-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:19:32.123 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-common-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:19:32.123 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-radosgw-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:19:32.123 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-selinux-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:19:32.123 INFO:teuthology.orchestra.run.smithi078.stdout: fmt-8.1.1-5.el9.x86_64 2024-09-21T09:19:32.123 INFO:teuthology.orchestra.run.smithi078.stdout: gperftools-libs-2.9.1-3.el9.x86_64 2024-09-21T09:19:32.124 INFO:teuthology.orchestra.run.smithi078.stdout: libarrow-9.0.0-11.el9.x86_64 2024-09-21T09:19:32.124 INFO:teuthology.orchestra.run.smithi078.stdout: libarrow-doc-9.0.0-11.el9.noarch 2024-09-21T09:19:32.124 INFO:teuthology.orchestra.run.smithi078.stdout: libcephfs2-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:19:32.124 INFO:teuthology.orchestra.run.smithi078.stdout: liboath-2.6.7-2.el9.x86_64 2024-09-21T09:19:32.124 INFO:teuthology.orchestra.run.smithi078.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-09-21T09:19:32.124 INFO:teuthology.orchestra.run.smithi078.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-09-21T09:19:32.124 INFO:teuthology.orchestra.run.smithi078.stdout: libradosstriper1-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:19:32.124 INFO:teuthology.orchestra.run.smithi078.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-09-21T09:19:32.124 INFO:teuthology.orchestra.run.smithi078.stdout: librgw2-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:19:32.124 INFO:teuthology.orchestra.run.smithi078.stdout: libunwind-1.6.2-1.el9.x86_64 2024-09-21T09:19:32.124 INFO:teuthology.orchestra.run.smithi078.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-09-21T09:19:32.124 INFO:teuthology.orchestra.run.smithi078.stdout: parquet-libs-9.0.0-11.el9.x86_64 2024-09-21T09:19:32.124 INFO:teuthology.orchestra.run.smithi078.stdout: python3-ceph-argparse-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:19:32.124 INFO:teuthology.orchestra.run.smithi078.stdout: python3-ceph-common-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:19:32.125 INFO:teuthology.orchestra.run.smithi078.stdout: python3-cephfs-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:19:32.125 INFO:teuthology.orchestra.run.smithi078.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-09-21T09:19:32.125 INFO:teuthology.orchestra.run.smithi078.stdout: python3-rados-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:19:32.125 INFO:teuthology.orchestra.run.smithi078.stdout: python3-rbd-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:19:32.125 INFO:teuthology.orchestra.run.smithi078.stdout: python3-rgw-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:19:32.125 INFO:teuthology.orchestra.run.smithi078.stdout: re2-1:20211101-20.el9.x86_64 2024-09-21T09:19:32.125 INFO:teuthology.orchestra.run.smithi078.stdout: thrift-0.15.0-2.el9.x86_64 2024-09-21T09:19:32.125 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:19:32.125 INFO:teuthology.orchestra.run.smithi078.stdout:Complete! 2024-09-21T09:19:32.476 DEBUG:teuthology.orchestra.run.smithi078:> sudo yum -y install ceph-test 2024-09-21T09:19:33.057 INFO:teuthology.orchestra.run.smithi078.stdout:Last metadata expiration check: 0:00:27 ago on Sat 21 Sep 2024 09:19:06 AM UTC. 2024-09-21T09:19:33.187 INFO:teuthology.orchestra.run.smithi078.stdout:Dependencies resolved. 2024-09-21T09:19:33.188 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-21T09:19:33.188 INFO:teuthology.orchestra.run.smithi078.stdout: Package Arch Version Repository Size 2024-09-21T09:19:33.188 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-21T09:19:33.188 INFO:teuthology.orchestra.run.smithi078.stdout:Installing: 2024-09-21T09:19:33.188 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-test x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 40 M 2024-09-21T09:19:33.188 INFO:teuthology.orchestra.run.smithi078.stdout:Installing dependencies: 2024-09-21T09:19:33.188 INFO:teuthology.orchestra.run.smithi078.stdout: libcephsqlite x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 166 k 2024-09-21T09:19:33.188 INFO:teuthology.orchestra.run.smithi078.stdout: socat x86_64 1.7.4.1-6.el9 appstream 304 k 2024-09-21T09:19:33.188 INFO:teuthology.orchestra.run.smithi078.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2024-09-21T09:19:33.189 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:19:33.189 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction Summary 2024-09-21T09:19:33.189 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-21T09:19:33.189 INFO:teuthology.orchestra.run.smithi078.stdout:Install 4 Packages 2024-09-21T09:19:33.189 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:19:33.189 INFO:teuthology.orchestra.run.smithi078.stdout:Total download size: 40 M 2024-09-21T09:19:33.189 INFO:teuthology.orchestra.run.smithi078.stdout:Installed size: 164 M 2024-09-21T09:19:33.190 INFO:teuthology.orchestra.run.smithi078.stdout:Downloading Packages: 2024-09-21T09:19:33.429 INFO:teuthology.orchestra.run.smithi078.stdout:(1/4): socat-1.7.4.1-6.el9.x86_64.rpm 2.2 MB/s | 304 kB 00:00 2024-09-21T09:19:33.462 INFO:teuthology.orchestra.run.smithi078.stdout:(2/4): xmlstarlet-1.6.1-20.el9.x86_64.rpm 1.9 MB/s | 64 kB 00:00 2024-09-21T09:19:33.504 INFO:teuthology.orchestra.run.smithi078.stdout:(3/4): libcephsqlite-18.2.4-871.g6a1bdf85.el9.x 783 kB/s | 166 kB 00:00 2024-09-21T09:19:34.563 INFO:teuthology.orchestra.run.smithi078.stdout:(4/4): ceph-test-18.2.4-871.g6a1bdf85.el9.x86_6 31 MB/s | 40 MB 00:01 2024-09-21T09:19:34.565 INFO:teuthology.orchestra.run.smithi078.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:19:34.565 INFO:teuthology.orchestra.run.smithi078.stdout:Total 29 MB/s | 40 MB 00:01 2024-09-21T09:19:34.594 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction check 2024-09-21T09:19:34.611 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction check succeeded. 2024-09-21T09:19:34.611 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction test 2024-09-21T09:19:34.926 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction test succeeded. 2024-09-21T09:19:34.926 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction 2024-09-21T09:19:35.393 INFO:teuthology.orchestra.run.smithi078.stdout: Preparing : 1/1 2024-09-21T09:19:35.492 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 1/4 2024-09-21T09:19:35.537 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : socat-1.7.4.1-6.el9.x86_64 2/4 2024-09-21T09:19:35.565 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : libcephsqlite-2:18.2.4-871.g6a1bdf85.el9.x86_64 3/4 2024-09-21T09:19:39.249 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: libcephsqlite-2:18.2.4-871.g6a1bdf85.el9.x86_64 3/4 2024-09-21T09:19:39.329 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : ceph-test-2:18.2.4-871.g6a1bdf85.el9.x86_64 4/4 2024-09-21T09:19:41.323 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: ceph-test-2:18.2.4-871.g6a1bdf85.el9.x86_64 4/4 2024-09-21T09:19:41.324 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : ceph-test-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/4 2024-09-21T09:19:41.324 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : libcephsqlite-2:18.2.4-871.g6a1bdf85.el9.x86_64 2/4 2024-09-21T09:19:41.324 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : socat-1.7.4.1-6.el9.x86_64 3/4 2024-09-21T09:19:41.659 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 4/4 2024-09-21T09:19:41.659 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:19:41.660 INFO:teuthology.orchestra.run.smithi078.stdout:Installed: 2024-09-21T09:19:41.660 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-test-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:19:41.660 INFO:teuthology.orchestra.run.smithi078.stdout: libcephsqlite-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:19:41.660 INFO:teuthology.orchestra.run.smithi078.stdout: socat-1.7.4.1-6.el9.x86_64 2024-09-21T09:19:41.660 INFO:teuthology.orchestra.run.smithi078.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-09-21T09:19:41.660 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:19:41.660 INFO:teuthology.orchestra.run.smithi078.stdout:Complete! 2024-09-21T09:19:41.953 DEBUG:teuthology.orchestra.run.smithi078:> sudo yum -y install ceph 2024-09-21T09:19:42.538 INFO:teuthology.orchestra.run.smithi078.stdout:Last metadata expiration check: 0:00:36 ago on Sat 21 Sep 2024 09:19:06 AM UTC. 2024-09-21T09:19:42.678 INFO:teuthology.orchestra.run.smithi078.stdout:Dependencies resolved. 2024-09-21T09:19:42.681 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-21T09:19:42.681 INFO:teuthology.orchestra.run.smithi078.stdout: Package Arch Version Repository Size 2024-09-21T09:19:42.681 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-21T09:19:42.681 INFO:teuthology.orchestra.run.smithi078.stdout:Installing: 2024-09-21T09:19:42.681 INFO:teuthology.orchestra.run.smithi078.stdout: ceph x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 6.5 k 2024-09-21T09:19:42.682 INFO:teuthology.orchestra.run.smithi078.stdout:Installing dependencies: 2024-09-21T09:19:42.682 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-mds x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 2.1 M 2024-09-21T09:19:42.682 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-mgr x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 1.5 M 2024-09-21T09:19:42.682 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-mgr-modules-core noarch 2:18.2.4-871.g6a1bdf85.el9 ceph-noarch 246 k 2024-09-21T09:19:42.682 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-mon x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 4.7 M 2024-09-21T09:19:42.682 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-osd x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 17 M 2024-09-21T09:19:42.682 INFO:teuthology.orchestra.run.smithi078.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-09-21T09:19:42.682 INFO:teuthology.orchestra.run.smithi078.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2024-09-21T09:19:42.682 INFO:teuthology.orchestra.run.smithi078.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-09-21T09:19:42.682 INFO:teuthology.orchestra.run.smithi078.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-09-21T09:19:42.682 INFO:teuthology.orchestra.run.smithi078.stdout: python3-cryptography x86_64 36.0.1-4.el9 baseos 1.2 M 2024-09-21T09:19:42.683 INFO:teuthology.orchestra.run.smithi078.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-09-21T09:19:42.683 INFO:teuthology.orchestra.run.smithi078.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-09-21T09:19:42.683 INFO:teuthology.orchestra.run.smithi078.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-09-21T09:19:42.683 INFO:teuthology.orchestra.run.smithi078.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-09-21T09:19:42.683 INFO:teuthology.orchestra.run.smithi078.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-09-21T09:19:42.683 INFO:teuthology.orchestra.run.smithi078.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-09-21T09:19:42.683 INFO:teuthology.orchestra.run.smithi078.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2024-09-21T09:19:42.683 INFO:teuthology.orchestra.run.smithi078.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2024-09-21T09:19:42.683 INFO:teuthology.orchestra.run.smithi078.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-09-21T09:19:42.683 INFO:teuthology.orchestra.run.smithi078.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-09-21T09:19:42.684 INFO:teuthology.orchestra.run.smithi078.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2024-09-21T09:19:42.684 INFO:teuthology.orchestra.run.smithi078.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-09-21T09:19:42.684 INFO:teuthology.orchestra.run.smithi078.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-09-21T09:19:42.684 INFO:teuthology.orchestra.run.smithi078.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2024-09-21T09:19:42.684 INFO:teuthology.orchestra.run.smithi078.stdout: python3-pytz noarch 2021.1-5.el9 appstream 51 k 2024-09-21T09:19:42.684 INFO:teuthology.orchestra.run.smithi078.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-09-21T09:19:42.684 INFO:teuthology.orchestra.run.smithi078.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-09-21T09:19:42.684 INFO:teuthology.orchestra.run.smithi078.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2024-09-21T09:19:42.684 INFO:teuthology.orchestra.run.smithi078.stdout: python3-urllib3 noarch 1.26.5-6.el9 baseos 215 k 2024-09-21T09:19:42.684 INFO:teuthology.orchestra.run.smithi078.stdout: python3-webob noarch 1.8.8-2.el9 epel 230 k 2024-09-21T09:19:42.684 INFO:teuthology.orchestra.run.smithi078.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-09-21T09:19:42.685 INFO:teuthology.orchestra.run.smithi078.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-09-21T09:19:42.685 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:19:42.685 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction Summary 2024-09-21T09:19:42.685 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-21T09:19:42.685 INFO:teuthology.orchestra.run.smithi078.stdout:Install 33 Packages 2024-09-21T09:19:42.685 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:19:42.686 INFO:teuthology.orchestra.run.smithi078.stdout:Total download size: 29 M 2024-09-21T09:19:42.686 INFO:teuthology.orchestra.run.smithi078.stdout:Installed size: 104 M 2024-09-21T09:19:42.686 INFO:teuthology.orchestra.run.smithi078.stdout:Downloading Packages: 2024-09-21T09:19:43.389 INFO:teuthology.orchestra.run.smithi078.stdout:(1/33): ceph-18.2.4-871.g6a1bdf85.el9.x86_64.rp 51 kB/s | 6.5 kB 00:00 2024-09-21T09:19:43.606 INFO:teuthology.orchestra.run.smithi078.stdout:(2/33): ceph-mds-18.2.4-871.g6a1bdf85.el9.x86_6 6.1 MB/s | 2.1 MB 00:00 2024-09-21T09:19:43.639 INFO:teuthology.orchestra.run.smithi078.stdout:(3/33): ceph-mgr-18.2.4-871.g6a1bdf85.el9.x86_6 3.9 MB/s | 1.5 MB 00:00 2024-09-21T09:19:43.764 INFO:teuthology.orchestra.run.smithi078.stdout:(4/33): ceph-mon-18.2.4-871.g6a1bdf85.el9.x86_6 12 MB/s | 4.7 MB 00:00 2024-09-21T09:19:43.789 INFO:teuthology.orchestra.run.smithi078.stdout:(5/33): ceph-mgr-modules-core-18.2.4-871.g6a1bd 1.6 MB/s | 246 kB 00:00 2024-09-21T09:19:43.973 INFO:teuthology.orchestra.run.smithi078.stdout:(6/33): python3-cffi-1.14.5-5.el9.x86_64.rpm 1.2 MB/s | 253 kB 00:00 2024-09-21T09:19:44.149 INFO:teuthology.orchestra.run.smithi078.stdout:(7/33): ceph-osd-18.2.4-871.g6a1bdf85.el9.x86_6 30 MB/s | 17 MB 00:00 2024-09-21T09:19:44.174 INFO:teuthology.orchestra.run.smithi078.stdout:(8/33): python3-ply-3.11-14.el9.noarch.rpm 533 kB/s | 106 kB 00:00 2024-09-21T09:19:44.232 INFO:teuthology.orchestra.run.smithi078.stdout:(9/33): python3-requests-2.25.1-8.el9.noarch.rp 2.1 MB/s | 125 kB 00:00 2024-09-21T09:19:44.266 INFO:teuthology.orchestra.run.smithi078.stdout:(10/33): python3-cryptography-36.0.1-4.el9.x86_ 2.6 MB/s | 1.2 MB 00:00 2024-09-21T09:19:44.325 INFO:teuthology.orchestra.run.smithi078.stdout:(11/33): python3-urllib3-1.26.5-6.el9.noarch.rp 2.3 MB/s | 215 kB 00:00 2024-09-21T09:19:44.375 INFO:teuthology.orchestra.run.smithi078.stdout:(12/33): python3-pycparser-2.20-6.el9.noarch.rp 598 kB/s | 135 kB 00:00 2024-09-21T09:19:44.425 INFO:teuthology.orchestra.run.smithi078.stdout:(13/33): python3-markupsafe-1.1.1-12.el9.x86_64 347 kB/s | 35 kB 00:00 2024-09-21T09:19:44.450 INFO:teuthology.orchestra.run.smithi078.stdout:(14/33): python3-mako-1.1.4-6.el9.noarch.rpm 935 kB/s | 172 kB 00:00 2024-09-21T09:19:44.484 INFO:teuthology.orchestra.run.smithi078.stdout:(15/33): python3-pytz-2021.1-5.el9.noarch.rpm 469 kB/s | 51 kB 00:00 2024-09-21T09:19:44.509 INFO:teuthology.orchestra.run.smithi078.stdout:(16/33): python3-toml-0.10.2-6.el9.noarch.rpm 497 kB/s | 42 kB 00:00 2024-09-21T09:19:44.543 INFO:teuthology.orchestra.run.smithi078.stdout:(17/33): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 473 kB/s | 43 kB 00:00 2024-09-21T09:19:44.576 INFO:teuthology.orchestra.run.smithi078.stdout:(18/33): python3-jaraco-8.2.1-3.el9.noarch.rpm 441 kB/s | 11 kB 00:00 2024-09-21T09:19:44.601 INFO:teuthology.orchestra.run.smithi078.stdout:(19/33): python3-cheroot-8.6.0-4.el9.noarch.rpm 1.5 MB/s | 172 kB 00:00 2024-09-21T09:19:44.627 INFO:teuthology.orchestra.run.smithi078.stdout:(20/33): python3-jaraco-classes-3.2.1-5.el9.noa 365 kB/s | 18 kB 00:00 2024-09-21T09:19:44.652 INFO:teuthology.orchestra.run.smithi078.stdout:(21/33): python3-jaraco-collections-3.0.0-8.el9 460 kB/s | 23 kB 00:00 2024-09-21T09:19:44.677 INFO:teuthology.orchestra.run.smithi078.stdout:(22/33): python3-jaraco-functools-3.5.0-2.el9.n 387 kB/s | 19 kB 00:00 2024-09-21T09:19:44.703 INFO:teuthology.orchestra.run.smithi078.stdout:(23/33): python3-jaraco-text-3.2.0-6.el9.noarch 392 kB/s | 20 kB 00:00 2024-09-21T09:19:44.728 INFO:teuthology.orchestra.run.smithi078.stdout:(24/33): python3-logutils-0.3.5-21.el9.noarch.r 924 kB/s | 46 kB 00:00 2024-09-21T09:19:44.753 INFO:teuthology.orchestra.run.smithi078.stdout:(25/33): python3-more-itertools-8.12.0-2.el9.no 1.5 MB/s | 79 kB 00:00 2024-09-21T09:19:44.779 INFO:teuthology.orchestra.run.smithi078.stdout:(26/33): python3-portend-3.1.0-2.el9.noarch.rpm 656 kB/s | 16 kB 00:00 2024-09-21T09:19:44.804 INFO:teuthology.orchestra.run.smithi078.stdout:(27/33): python3-cherrypy-18.6.1-2.el9.noarch.r 1.2 MB/s | 358 kB 00:00 2024-09-21T09:19:44.829 INFO:teuthology.orchestra.run.smithi078.stdout:(28/33): python3-pyOpenSSL-21.0.0-1.el9.noarch. 1.8 MB/s | 90 kB 00:00 2024-09-21T09:19:44.854 INFO:teuthology.orchestra.run.smithi078.stdout:(29/33): python3-tempora-5.0.0-2.el9.noarch.rpm 716 kB/s | 36 kB 00:00 2024-09-21T09:19:44.888 INFO:teuthology.orchestra.run.smithi078.stdout:(30/33): python3-pecan-1.4.2-3.el9.noarch.rpm 1.7 MB/s | 272 kB 00:00 2024-09-21T09:19:44.913 INFO:teuthology.orchestra.run.smithi078.stdout:(31/33): python3-zc-lockfile-2.0-10.el9.noarch. 806 kB/s | 20 kB 00:00 2024-09-21T09:19:44.947 INFO:teuthology.orchestra.run.smithi078.stdout:(32/33): python3-webob-1.8.8-2.el9.noarch.rpm 2.4 MB/s | 230 kB 00:00 2024-09-21T09:19:44.980 INFO:teuthology.orchestra.run.smithi078.stdout:(33/33): python3-werkzeug-2.0.3-3.el9.1.noarch. 3.3 MB/s | 427 kB 00:00 2024-09-21T09:19:44.985 INFO:teuthology.orchestra.run.smithi078.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:19:44.985 INFO:teuthology.orchestra.run.smithi078.stdout:Total 13 MB/s | 29 MB 00:02 2024-09-21T09:19:45.108 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction check 2024-09-21T09:19:45.145 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction check succeeded. 2024-09-21T09:19:45.145 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction test 2024-09-21T09:19:45.559 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction test succeeded. 2024-09-21T09:19:45.559 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction 2024-09-21T09:19:46.775 INFO:teuthology.orchestra.run.smithi078.stdout: Preparing : 1/1 2024-09-21T09:19:46.842 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/33 2024-09-21T09:19:46.874 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/33 2024-09-21T09:19:46.915 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 3/33 2024-09-21T09:19:47.006 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 4/33 2024-09-21T09:19:47.093 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 5/33 2024-09-21T09:19:47.135 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-webob-1.8.8-2.el9.noarch 6/33 2024-09-21T09:19:47.187 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 7/33 2024-09-21T09:19:47.224 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 8/33 2024-09-21T09:19:47.259 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 9/33 2024-09-21T09:19:47.292 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 10/33 2024-09-21T09:19:47.332 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 11/33 2024-09-21T09:19:47.380 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 12/33 2024-09-21T09:19:47.422 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 13/33 2024-09-21T09:19:47.460 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 14/33 2024-09-21T09:19:47.533 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 15/33 2024-09-21T09:19:47.679 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 16/33 2024-09-21T09:19:47.759 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 17/33 2024-09-21T09:19:47.813 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-urllib3-1.26.5-6.el9.noarch 18/33 2024-09-21T09:19:47.852 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 19/33 2024-09-21T09:19:47.909 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-ply-3.11-14.el9.noarch 20/33 2024-09-21T09:19:47.973 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 21/33 2024-09-21T09:19:48.182 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 22/33 2024-09-21T09:19:48.239 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 23/33 2024-09-21T09:19:48.316 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 24/33 2024-09-21T09:19:48.405 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 25/33 2024-09-21T09:19:48.446 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 26/33 2024-09-21T09:19:48.570 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 27/33 2024-09-21T09:19:48.734 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : ceph-mgr-modules-core-2:18.2.4-871.g6a1bdf85.el9.n 28/33 2024-09-21T09:19:48.766 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : ceph-mgr-2:18.2.4-871.g6a1bdf85.el9.x86_64 29/33 2024-09-21T09:19:48.791 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: ceph-mgr-2:18.2.4-871.g6a1bdf85.el9.x86_64 29/33 2024-09-21T09:19:48.791 INFO:teuthology.orchestra.run.smithi078.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:19:48.791 INFO:teuthology.orchestra.run.smithi078.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-09-21T09:19:48.791 INFO:teuthology.orchestra.run.smithi078.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-09-21T09:19:48.791 INFO:teuthology.orchestra.run.smithi078.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-09-21T09:19:48.791 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:19:50.210 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : ceph-osd-2:18.2.4-871.g6a1bdf85.el9.x86_64 30/33 2024-09-21T09:19:50.240 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: ceph-osd-2:18.2.4-871.g6a1bdf85.el9.x86_64 30/33 2024-09-21T09:19:50.240 INFO:teuthology.orchestra.run.smithi078.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:19:50.240 INFO:teuthology.orchestra.run.smithi078.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-09-21T09:19:50.240 INFO:teuthology.orchestra.run.smithi078.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-09-21T09:19:50.240 INFO:teuthology.orchestra.run.smithi078.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-09-21T09:19:50.240 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:19:50.702 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : ceph-mon-2:18.2.4-871.g6a1bdf85.el9.x86_64 31/33 2024-09-21T09:19:50.727 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: ceph-mon-2:18.2.4-871.g6a1bdf85.el9.x86_64 31/33 2024-09-21T09:19:50.727 INFO:teuthology.orchestra.run.smithi078.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:19:50.727 INFO:teuthology.orchestra.run.smithi078.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-09-21T09:19:50.727 INFO:teuthology.orchestra.run.smithi078.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-09-21T09:19:50.728 INFO:teuthology.orchestra.run.smithi078.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-09-21T09:19:50.728 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:19:51.164 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : ceph-mds-2:18.2.4-871.g6a1bdf85.el9.x86_64 32/33 2024-09-21T09:19:51.190 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: ceph-mds-2:18.2.4-871.g6a1bdf85.el9.x86_64 32/33 2024-09-21T09:19:51.190 INFO:teuthology.orchestra.run.smithi078.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:19:51.190 INFO:teuthology.orchestra.run.smithi078.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-09-21T09:19:51.190 INFO:teuthology.orchestra.run.smithi078.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-09-21T09:19:51.190 INFO:teuthology.orchestra.run.smithi078.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-09-21T09:19:51.190 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:19:51.275 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : ceph-2:18.2.4-871.g6a1bdf85.el9.x86_64 33/33 2024-09-21T09:19:52.859 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: ceph-2:18.2.4-871.g6a1bdf85.el9.x86_64 33/33 2024-09-21T09:19:52.859 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : ceph-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/33 2024-09-21T09:19:52.859 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : ceph-mds-2:18.2.4-871.g6a1bdf85.el9.x86_64 2/33 2024-09-21T09:19:52.859 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : ceph-mgr-2:18.2.4-871.g6a1bdf85.el9.x86_64 3/33 2024-09-21T09:19:52.859 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : ceph-mon-2:18.2.4-871.g6a1bdf85.el9.x86_64 4/33 2024-09-21T09:19:52.859 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : ceph-osd-2:18.2.4-871.g6a1bdf85.el9.x86_64 5/33 2024-09-21T09:19:52.859 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : ceph-mgr-modules-core-2:18.2.4-871.g6a1bdf85.el9.n 6/33 2024-09-21T09:19:52.859 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 7/33 2024-09-21T09:19:52.859 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 8/33 2024-09-21T09:19:52.859 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-ply-3.11-14.el9.noarch 9/33 2024-09-21T09:19:52.859 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 10/33 2024-09-21T09:19:52.859 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 11/33 2024-09-21T09:19:52.860 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-urllib3-1.26.5-6.el9.noarch 12/33 2024-09-21T09:19:52.860 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 13/33 2024-09-21T09:19:52.860 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 14/33 2024-09-21T09:19:52.860 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 15/33 2024-09-21T09:19:52.860 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 16/33 2024-09-21T09:19:52.860 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 17/33 2024-09-21T09:19:52.860 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 18/33 2024-09-21T09:19:52.860 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 19/33 2024-09-21T09:19:52.860 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 20/33 2024-09-21T09:19:52.860 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 21/33 2024-09-21T09:19:52.860 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 22/33 2024-09-21T09:19:52.860 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 23/33 2024-09-21T09:19:52.860 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 24/33 2024-09-21T09:19:52.861 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 25/33 2024-09-21T09:19:52.861 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 26/33 2024-09-21T09:19:52.862 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 27/33 2024-09-21T09:19:52.862 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 28/33 2024-09-21T09:19:52.862 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 29/33 2024-09-21T09:19:52.862 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 30/33 2024-09-21T09:19:52.862 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-webob-1.8.8-2.el9.noarch 31/33 2024-09-21T09:19:52.862 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 32/33 2024-09-21T09:19:53.887 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 33/33 2024-09-21T09:19:53.887 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:19:53.887 INFO:teuthology.orchestra.run.smithi078.stdout:Installed: 2024-09-21T09:19:53.888 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:19:53.888 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-mds-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:19:53.888 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-mgr-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:19:53.888 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-mgr-modules-core-2:18.2.4-871.g6a1bdf85.el9.noarch 2024-09-21T09:19:53.888 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-mon-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:19:53.888 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-osd-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:19:53.888 INFO:teuthology.orchestra.run.smithi078.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-09-21T09:19:53.888 INFO:teuthology.orchestra.run.smithi078.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-09-21T09:19:53.888 INFO:teuthology.orchestra.run.smithi078.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-09-21T09:19:53.888 INFO:teuthology.orchestra.run.smithi078.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-09-21T09:19:53.888 INFO:teuthology.orchestra.run.smithi078.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-09-21T09:19:53.889 INFO:teuthology.orchestra.run.smithi078.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-09-21T09:19:53.889 INFO:teuthology.orchestra.run.smithi078.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-09-21T09:19:53.889 INFO:teuthology.orchestra.run.smithi078.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-09-21T09:19:53.889 INFO:teuthology.orchestra.run.smithi078.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-09-21T09:19:53.889 INFO:teuthology.orchestra.run.smithi078.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-09-21T09:19:53.889 INFO:teuthology.orchestra.run.smithi078.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-09-21T09:19:53.889 INFO:teuthology.orchestra.run.smithi078.stdout: python3-mako-1.1.4-6.el9.noarch 2024-09-21T09:19:53.889 INFO:teuthology.orchestra.run.smithi078.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-09-21T09:19:53.889 INFO:teuthology.orchestra.run.smithi078.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-09-21T09:19:53.889 INFO:teuthology.orchestra.run.smithi078.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-09-21T09:19:53.889 INFO:teuthology.orchestra.run.smithi078.stdout: python3-ply-3.11-14.el9.noarch 2024-09-21T09:19:53.889 INFO:teuthology.orchestra.run.smithi078.stdout: python3-portend-3.1.0-2.el9.noarch 2024-09-21T09:19:53.890 INFO:teuthology.orchestra.run.smithi078.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-09-21T09:19:53.890 INFO:teuthology.orchestra.run.smithi078.stdout: python3-pycparser-2.20-6.el9.noarch 2024-09-21T09:19:53.890 INFO:teuthology.orchestra.run.smithi078.stdout: python3-pytz-2021.1-5.el9.noarch 2024-09-21T09:19:53.890 INFO:teuthology.orchestra.run.smithi078.stdout: python3-requests-2.25.1-8.el9.noarch 2024-09-21T09:19:53.890 INFO:teuthology.orchestra.run.smithi078.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-09-21T09:19:53.890 INFO:teuthology.orchestra.run.smithi078.stdout: python3-toml-0.10.2-6.el9.noarch 2024-09-21T09:19:53.890 INFO:teuthology.orchestra.run.smithi078.stdout: python3-urllib3-1.26.5-6.el9.noarch 2024-09-21T09:19:53.890 INFO:teuthology.orchestra.run.smithi078.stdout: python3-webob-1.8.8-2.el9.noarch 2024-09-21T09:19:53.890 INFO:teuthology.orchestra.run.smithi078.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-09-21T09:19:53.890 INFO:teuthology.orchestra.run.smithi078.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-09-21T09:19:53.890 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:19:53.890 INFO:teuthology.orchestra.run.smithi078.stdout:Complete! 2024-09-21T09:19:54.311 DEBUG:teuthology.orchestra.run.smithi078:> sudo yum -y install ceph-base 2024-09-21T09:19:54.897 INFO:teuthology.orchestra.run.smithi078.stdout:Last metadata expiration check: 0:00:48 ago on Sat 21 Sep 2024 09:19:06 AM UTC. 2024-09-21T09:19:54.986 INFO:teuthology.orchestra.run.smithi078.stdout:Package ceph-base-2:18.2.4-871.g6a1bdf85.el9.x86_64 is already installed. 2024-09-21T09:19:55.028 INFO:teuthology.orchestra.run.smithi078.stdout:Dependencies resolved. 2024-09-21T09:19:55.029 INFO:teuthology.orchestra.run.smithi078.stdout:Nothing to do. 2024-09-21T09:19:55.029 INFO:teuthology.orchestra.run.smithi078.stdout:Complete! 2024-09-21T09:19:55.103 DEBUG:teuthology.orchestra.run.smithi078:> sudo yum -y install ceph-immutable-object-cache 2024-09-21T09:19:55.681 INFO:teuthology.orchestra.run.smithi078.stdout:Last metadata expiration check: 0:00:49 ago on Sat 21 Sep 2024 09:19:06 AM UTC. 2024-09-21T09:19:55.812 INFO:teuthology.orchestra.run.smithi078.stdout:Dependencies resolved. 2024-09-21T09:19:55.813 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-21T09:19:55.813 INFO:teuthology.orchestra.run.smithi078.stdout: Package Arch Version Repo Size 2024-09-21T09:19:55.814 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-21T09:19:55.814 INFO:teuthology.orchestra.run.smithi078.stdout:Installing: 2024-09-21T09:19:55.814 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-immutable-object-cache x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 142 k 2024-09-21T09:19:55.814 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:19:55.814 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction Summary 2024-09-21T09:19:55.814 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-21T09:19:55.814 INFO:teuthology.orchestra.run.smithi078.stdout:Install 1 Package 2024-09-21T09:19:55.814 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:19:55.814 INFO:teuthology.orchestra.run.smithi078.stdout:Total download size: 142 k 2024-09-21T09:19:55.814 INFO:teuthology.orchestra.run.smithi078.stdout:Installed size: 395 k 2024-09-21T09:19:55.814 INFO:teuthology.orchestra.run.smithi078.stdout:Downloading Packages: 2024-09-21T09:19:56.026 INFO:teuthology.orchestra.run.smithi078.stdout:ceph-immutable-object-cache-18.2.4-871.g6a1bdf8 673 kB/s | 142 kB 00:00 2024-09-21T09:19:56.027 INFO:teuthology.orchestra.run.smithi078.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:19:56.027 INFO:teuthology.orchestra.run.smithi078.stdout:Total 669 kB/s | 142 kB 00:00 2024-09-21T09:19:56.027 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction check 2024-09-21T09:19:56.034 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction check succeeded. 2024-09-21T09:19:56.035 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction test 2024-09-21T09:19:56.091 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction test succeeded. 2024-09-21T09:19:56.091 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction 2024-09-21T09:19:56.243 INFO:teuthology.orchestra.run.smithi078.stdout: Preparing : 1/1 2024-09-21T09:19:56.302 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : ceph-immutable-object-cache-2:18.2.4-871.g6a1bdf85.e 1/1 2024-09-21T09:19:56.328 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: ceph-immutable-object-cache-2:18.2.4-871.g6a1bdf85.e 1/1 2024-09-21T09:19:56.328 INFO:teuthology.orchestra.run.smithi078.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:19:56.328 INFO:teuthology.orchestra.run.smithi078.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-09-21T09:19:56.328 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:19:57.299 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : ceph-immutable-object-cache-2:18.2.4-871.g6a1bdf85.e 1/1 2024-09-21T09:19:57.299 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:19:57.300 INFO:teuthology.orchestra.run.smithi078.stdout:Installed: 2024-09-21T09:19:57.300 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-immutable-object-cache-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:19:57.300 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:19:57.300 INFO:teuthology.orchestra.run.smithi078.stdout:Complete! 2024-09-21T09:19:57.546 DEBUG:teuthology.orchestra.run.smithi078:> sudo yum -y install ceph-mgr 2024-09-21T09:19:58.143 INFO:teuthology.orchestra.run.smithi078.stdout:Last metadata expiration check: 0:00:52 ago on Sat 21 Sep 2024 09:19:06 AM UTC. 2024-09-21T09:19:58.231 INFO:teuthology.orchestra.run.smithi078.stdout:Package ceph-mgr-2:18.2.4-871.g6a1bdf85.el9.x86_64 is already installed. 2024-09-21T09:19:58.272 INFO:teuthology.orchestra.run.smithi078.stdout:Dependencies resolved. 2024-09-21T09:19:58.273 INFO:teuthology.orchestra.run.smithi078.stdout:Nothing to do. 2024-09-21T09:19:58.273 INFO:teuthology.orchestra.run.smithi078.stdout:Complete! 2024-09-21T09:19:58.338 DEBUG:teuthology.orchestra.run.smithi078:> sudo yum -y install ceph-fuse 2024-09-21T09:19:58.916 INFO:teuthology.orchestra.run.smithi078.stdout:Last metadata expiration check: 0:00:52 ago on Sat 21 Sep 2024 09:19:06 AM UTC. 2024-09-21T09:19:59.047 INFO:teuthology.orchestra.run.smithi078.stdout:Dependencies resolved. 2024-09-21T09:19:59.047 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-21T09:19:59.047 INFO:teuthology.orchestra.run.smithi078.stdout: Package Arch Version Repository Size 2024-09-21T09:19:59.047 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-21T09:19:59.047 INFO:teuthology.orchestra.run.smithi078.stdout:Installing: 2024-09-21T09:19:59.047 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-fuse x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 853 k 2024-09-21T09:19:59.047 INFO:teuthology.orchestra.run.smithi078.stdout:Installing dependencies: 2024-09-21T09:19:59.048 INFO:teuthology.orchestra.run.smithi078.stdout: fuse x86_64 2.9.9-16.el9 baseos 80 k 2024-09-21T09:19:59.048 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:19:59.048 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction Summary 2024-09-21T09:19:59.048 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-21T09:19:59.048 INFO:teuthology.orchestra.run.smithi078.stdout:Install 2 Packages 2024-09-21T09:19:59.048 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:19:59.048 INFO:teuthology.orchestra.run.smithi078.stdout:Total download size: 932 k 2024-09-21T09:19:59.049 INFO:teuthology.orchestra.run.smithi078.stdout:Installed size: 2.7 M 2024-09-21T09:19:59.049 INFO:teuthology.orchestra.run.smithi078.stdout:Downloading Packages: 2024-09-21T09:19:59.346 INFO:teuthology.orchestra.run.smithi078.stdout:(1/2): fuse-2.9.9-16.el9.x86_64.rpm 407 kB/s | 80 kB 00:00 2024-09-21T09:19:59.421 INFO:teuthology.orchestra.run.smithi078.stdout:(2/2): ceph-fuse-18.2.4-871.g6a1bdf85.el9.x86_6 3.1 MB/s | 853 kB 00:00 2024-09-21T09:19:59.422 INFO:teuthology.orchestra.run.smithi078.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:19:59.422 INFO:teuthology.orchestra.run.smithi078.stdout:Total 2.4 MB/s | 932 kB 00:00 2024-09-21T09:19:59.434 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction check 2024-09-21T09:19:59.455 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction check succeeded. 2024-09-21T09:19:59.455 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction test 2024-09-21T09:19:59.540 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction test succeeded. 2024-09-21T09:19:59.541 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction 2024-09-21T09:19:59.731 INFO:teuthology.orchestra.run.smithi078.stdout: Preparing : 1/1 2024-09-21T09:19:59.880 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : fuse-2.9.9-16.el9.x86_64 1/2 2024-09-21T09:19:59.952 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : ceph-fuse-2:18.2.4-871.g6a1bdf85.el9.x86_64 2/2 2024-09-21T09:20:00.652 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: ceph-fuse-2:18.2.4-871.g6a1bdf85.el9.x86_64 2/2 2024-09-21T09:20:00.652 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : ceph-fuse-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/2 2024-09-21T09:20:00.917 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : fuse-2.9.9-16.el9.x86_64 2/2 2024-09-21T09:20:00.917 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:20:00.917 INFO:teuthology.orchestra.run.smithi078.stdout:Installed: 2024-09-21T09:20:00.917 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-fuse-2:18.2.4-871.g6a1bdf85.el9.x86_64 fuse-2.9.9-16.el9.x86_64 2024-09-21T09:20:00.917 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:20:00.917 INFO:teuthology.orchestra.run.smithi078.stdout:Complete! 2024-09-21T09:20:01.187 DEBUG:teuthology.orchestra.run.smithi078:> sudo yum -y install librados-devel 2024-09-21T09:20:01.780 INFO:teuthology.orchestra.run.smithi078.stdout:Last metadata expiration check: 0:00:55 ago on Sat 21 Sep 2024 09:19:06 AM UTC. 2024-09-21T09:20:01.913 INFO:teuthology.orchestra.run.smithi078.stdout:Dependencies resolved. 2024-09-21T09:20:01.913 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-21T09:20:01.913 INFO:teuthology.orchestra.run.smithi078.stdout: Package Arch Version Repo Size 2024-09-21T09:20:01.914 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-21T09:20:01.914 INFO:teuthology.orchestra.run.smithi078.stdout:Installing: 2024-09-21T09:20:01.914 INFO:teuthology.orchestra.run.smithi078.stdout: librados-devel x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 126 k 2024-09-21T09:20:01.914 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:20:01.914 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction Summary 2024-09-21T09:20:01.914 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-21T09:20:01.914 INFO:teuthology.orchestra.run.smithi078.stdout:Install 1 Package 2024-09-21T09:20:01.914 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:20:01.915 INFO:teuthology.orchestra.run.smithi078.stdout:Total download size: 126 k 2024-09-21T09:20:01.915 INFO:teuthology.orchestra.run.smithi078.stdout:Installed size: 456 k 2024-09-21T09:20:01.915 INFO:teuthology.orchestra.run.smithi078.stdout:Downloading Packages: 2024-09-21T09:20:02.128 INFO:teuthology.orchestra.run.smithi078.stdout:librados-devel-18.2.4-871.g6a1bdf85.el9.x86_64. 595 kB/s | 126 kB 00:00 2024-09-21T09:20:02.128 INFO:teuthology.orchestra.run.smithi078.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:20:02.129 INFO:teuthology.orchestra.run.smithi078.stdout:Total 590 kB/s | 126 kB 00:00 2024-09-21T09:20:02.129 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction check 2024-09-21T09:20:02.134 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction check succeeded. 2024-09-21T09:20:02.135 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction test 2024-09-21T09:20:02.191 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction test succeeded. 2024-09-21T09:20:02.191 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction 2024-09-21T09:20:02.335 INFO:teuthology.orchestra.run.smithi078.stdout: Preparing : 1/1 2024-09-21T09:20:02.448 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : librados-devel-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:20:02.827 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: librados-devel-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:20:03.090 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : librados-devel-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:20:03.090 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:20:03.090 INFO:teuthology.orchestra.run.smithi078.stdout:Installed: 2024-09-21T09:20:03.090 INFO:teuthology.orchestra.run.smithi078.stdout: librados-devel-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:20:03.090 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:20:03.090 INFO:teuthology.orchestra.run.smithi078.stdout:Complete! 2024-09-21T09:20:03.339 DEBUG:teuthology.orchestra.run.smithi078:> sudo yum -y install libcephfs2 2024-09-21T09:20:03.951 INFO:teuthology.orchestra.run.smithi078.stdout:Last metadata expiration check: 0:00:57 ago on Sat 21 Sep 2024 09:19:06 AM UTC. 2024-09-21T09:20:04.043 INFO:teuthology.orchestra.run.smithi078.stdout:Package libcephfs2-2:18.2.4-871.g6a1bdf85.el9.x86_64 is already installed. 2024-09-21T09:20:04.087 INFO:teuthology.orchestra.run.smithi078.stdout:Dependencies resolved. 2024-09-21T09:20:04.087 INFO:teuthology.orchestra.run.smithi078.stdout:Nothing to do. 2024-09-21T09:20:04.087 INFO:teuthology.orchestra.run.smithi078.stdout:Complete! 2024-09-21T09:20:04.157 DEBUG:teuthology.orchestra.run.smithi078:> sudo yum -y install libcephfs-devel 2024-09-21T09:20:04.732 INFO:teuthology.orchestra.run.smithi078.stdout:Last metadata expiration check: 0:00:58 ago on Sat 21 Sep 2024 09:19:06 AM UTC. 2024-09-21T09:20:04.866 INFO:teuthology.orchestra.run.smithi078.stdout:Dependencies resolved. 2024-09-21T09:20:04.867 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-21T09:20:04.867 INFO:teuthology.orchestra.run.smithi078.stdout: Package Arch Version Repo Size 2024-09-21T09:20:04.867 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-21T09:20:04.868 INFO:teuthology.orchestra.run.smithi078.stdout:Installing: 2024-09-21T09:20:04.868 INFO:teuthology.orchestra.run.smithi078.stdout: libcephfs-devel x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 31 k 2024-09-21T09:20:04.868 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:20:04.868 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction Summary 2024-09-21T09:20:04.868 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-21T09:20:04.868 INFO:teuthology.orchestra.run.smithi078.stdout:Install 1 Package 2024-09-21T09:20:04.868 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:20:04.868 INFO:teuthology.orchestra.run.smithi078.stdout:Total download size: 31 k 2024-09-21T09:20:04.868 INFO:teuthology.orchestra.run.smithi078.stdout:Installed size: 139 k 2024-09-21T09:20:04.868 INFO:teuthology.orchestra.run.smithi078.stdout:Downloading Packages: 2024-09-21T09:20:05.023 INFO:teuthology.orchestra.run.smithi078.stdout:libcephfs-devel-18.2.4-871.g6a1bdf85.el9.x86_64 201 kB/s | 31 kB 00:00 2024-09-21T09:20:05.091 INFO:teuthology.orchestra.run.smithi078.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:20:05.091 INFO:teuthology.orchestra.run.smithi078.stdout:Total 199 kB/s | 31 kB 00:00 2024-09-21T09:20:05.091 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction check 2024-09-21T09:20:05.092 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction check succeeded. 2024-09-21T09:20:05.092 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction test 2024-09-21T09:20:05.092 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction test succeeded. 2024-09-21T09:20:05.092 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction 2024-09-21T09:20:05.130 INFO:teuthology.orchestra.run.smithi078.stdout: Preparing : 1/1 2024-09-21T09:20:05.227 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : libcephfs-devel-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:20:05.596 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: libcephfs-devel-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:20:05.821 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : libcephfs-devel-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:20:05.822 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:20:05.822 INFO:teuthology.orchestra.run.smithi078.stdout:Installed: 2024-09-21T09:20:05.822 INFO:teuthology.orchestra.run.smithi078.stdout: libcephfs-devel-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:20:05.822 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:20:05.822 INFO:teuthology.orchestra.run.smithi078.stdout:Complete! 2024-09-21T09:20:05.995 DEBUG:teuthology.orchestra.run.smithi078:> sudo yum -y install librados2 2024-09-21T09:20:06.576 INFO:teuthology.orchestra.run.smithi078.stdout:Last metadata expiration check: 0:01:00 ago on Sat 21 Sep 2024 09:19:06 AM UTC. 2024-09-21T09:20:06.664 INFO:teuthology.orchestra.run.smithi078.stdout:Package librados2-2:18.2.4-871.g6a1bdf85.el9.x86_64 is already installed. 2024-09-21T09:20:06.706 INFO:teuthology.orchestra.run.smithi078.stdout:Dependencies resolved. 2024-09-21T09:20:06.707 INFO:teuthology.orchestra.run.smithi078.stdout:Nothing to do. 2024-09-21T09:20:06.707 INFO:teuthology.orchestra.run.smithi078.stdout:Complete! 2024-09-21T09:20:06.781 DEBUG:teuthology.orchestra.run.smithi078:> sudo yum -y install librbd1 2024-09-21T09:20:07.362 INFO:teuthology.orchestra.run.smithi078.stdout:Last metadata expiration check: 0:01:01 ago on Sat 21 Sep 2024 09:19:06 AM UTC. 2024-09-21T09:20:07.450 INFO:teuthology.orchestra.run.smithi078.stdout:Package librbd1-2:18.2.4-871.g6a1bdf85.el9.x86_64 is already installed. 2024-09-21T09:20:07.491 INFO:teuthology.orchestra.run.smithi078.stdout:Dependencies resolved. 2024-09-21T09:20:07.492 INFO:teuthology.orchestra.run.smithi078.stdout:Nothing to do. 2024-09-21T09:20:07.493 INFO:teuthology.orchestra.run.smithi078.stdout:Complete! 2024-09-21T09:20:07.564 DEBUG:teuthology.orchestra.run.smithi078:> sudo yum -y install python3-rados 2024-09-21T09:20:08.129 INFO:teuthology.orchestra.run.smithi078.stdout:Last metadata expiration check: 0:01:02 ago on Sat 21 Sep 2024 09:19:06 AM UTC. 2024-09-21T09:20:08.217 INFO:teuthology.orchestra.run.smithi078.stdout:Package python3-rados-2:18.2.4-871.g6a1bdf85.el9.x86_64 is already installed. 2024-09-21T09:20:08.259 INFO:teuthology.orchestra.run.smithi078.stdout:Dependencies resolved. 2024-09-21T09:20:08.260 INFO:teuthology.orchestra.run.smithi078.stdout:Nothing to do. 2024-09-21T09:20:08.260 INFO:teuthology.orchestra.run.smithi078.stdout:Complete! 2024-09-21T09:20:08.330 DEBUG:teuthology.orchestra.run.smithi078:> sudo yum -y install python3-rgw 2024-09-21T09:20:08.907 INFO:teuthology.orchestra.run.smithi078.stdout:Last metadata expiration check: 0:01:02 ago on Sat 21 Sep 2024 09:19:06 AM UTC. 2024-09-21T09:20:08.995 INFO:teuthology.orchestra.run.smithi078.stdout:Package python3-rgw-2:18.2.4-871.g6a1bdf85.el9.x86_64 is already installed. 2024-09-21T09:20:09.036 INFO:teuthology.orchestra.run.smithi078.stdout:Dependencies resolved. 2024-09-21T09:20:09.037 INFO:teuthology.orchestra.run.smithi078.stdout:Nothing to do. 2024-09-21T09:20:09.038 INFO:teuthology.orchestra.run.smithi078.stdout:Complete! 2024-09-21T09:20:09.111 DEBUG:teuthology.orchestra.run.smithi078:> sudo yum -y install python3-cephfs 2024-09-21T09:20:09.685 INFO:teuthology.orchestra.run.smithi078.stdout:Last metadata expiration check: 0:01:03 ago on Sat 21 Sep 2024 09:19:06 AM UTC. 2024-09-21T09:20:09.774 INFO:teuthology.orchestra.run.smithi078.stdout:Package python3-cephfs-2:18.2.4-871.g6a1bdf85.el9.x86_64 is already installed. 2024-09-21T09:20:09.815 INFO:teuthology.orchestra.run.smithi078.stdout:Dependencies resolved. 2024-09-21T09:20:09.816 INFO:teuthology.orchestra.run.smithi078.stdout:Nothing to do. 2024-09-21T09:20:09.816 INFO:teuthology.orchestra.run.smithi078.stdout:Complete! 2024-09-21T09:20:09.899 DEBUG:teuthology.orchestra.run.smithi078:> sudo yum -y install python3-rbd 2024-09-21T09:20:10.470 INFO:teuthology.orchestra.run.smithi078.stdout:Last metadata expiration check: 0:01:04 ago on Sat 21 Sep 2024 09:19:06 AM UTC. 2024-09-21T09:20:10.558 INFO:teuthology.orchestra.run.smithi078.stdout:Package python3-rbd-2:18.2.4-871.g6a1bdf85.el9.x86_64 is already installed. 2024-09-21T09:20:10.601 INFO:teuthology.orchestra.run.smithi078.stdout:Dependencies resolved. 2024-09-21T09:20:10.601 INFO:teuthology.orchestra.run.smithi078.stdout:Nothing to do. 2024-09-21T09:20:10.602 INFO:teuthology.orchestra.run.smithi078.stdout:Complete! 2024-09-21T09:20:10.673 DEBUG:teuthology.orchestra.run.smithi078:> sudo yum -y install rbd-fuse 2024-09-21T09:20:11.243 INFO:teuthology.orchestra.run.smithi078.stdout:Last metadata expiration check: 0:01:05 ago on Sat 21 Sep 2024 09:19:06 AM UTC. 2024-09-21T09:20:11.374 INFO:teuthology.orchestra.run.smithi078.stdout:Dependencies resolved. 2024-09-21T09:20:11.374 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-21T09:20:11.374 INFO:teuthology.orchestra.run.smithi078.stdout: Package Architecture Version Repository Size 2024-09-21T09:20:11.374 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-21T09:20:11.374 INFO:teuthology.orchestra.run.smithi078.stdout:Installing: 2024-09-21T09:20:11.374 INFO:teuthology.orchestra.run.smithi078.stdout: rbd-fuse x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 87 k 2024-09-21T09:20:11.374 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:20:11.374 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction Summary 2024-09-21T09:20:11.374 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-21T09:20:11.374 INFO:teuthology.orchestra.run.smithi078.stdout:Install 1 Package 2024-09-21T09:20:11.375 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:20:11.375 INFO:teuthology.orchestra.run.smithi078.stdout:Total download size: 87 k 2024-09-21T09:20:11.375 INFO:teuthology.orchestra.run.smithi078.stdout:Installed size: 227 k 2024-09-21T09:20:11.375 INFO:teuthology.orchestra.run.smithi078.stdout:Downloading Packages: 2024-09-21T09:20:11.576 INFO:teuthology.orchestra.run.smithi078.stdout:rbd-fuse-18.2.4-871.g6a1bdf85.el9.x86_64.rpm 434 kB/s | 87 kB 00:00 2024-09-21T09:20:11.577 INFO:teuthology.orchestra.run.smithi078.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:20:11.577 INFO:teuthology.orchestra.run.smithi078.stdout:Total 431 kB/s | 87 kB 00:00 2024-09-21T09:20:11.577 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction check 2024-09-21T09:20:11.584 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction check succeeded. 2024-09-21T09:20:11.584 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction test 2024-09-21T09:20:11.640 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction test succeeded. 2024-09-21T09:20:11.640 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction 2024-09-21T09:20:11.777 INFO:teuthology.orchestra.run.smithi078.stdout: Preparing : 1/1 2024-09-21T09:20:11.883 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : rbd-fuse-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:20:12.248 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: rbd-fuse-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:20:12.538 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : rbd-fuse-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:20:12.538 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:20:12.538 INFO:teuthology.orchestra.run.smithi078.stdout:Installed: 2024-09-21T09:20:12.538 INFO:teuthology.orchestra.run.smithi078.stdout: rbd-fuse-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:20:12.538 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:20:12.539 INFO:teuthology.orchestra.run.smithi078.stdout:Complete! 2024-09-21T09:20:12.786 DEBUG:teuthology.orchestra.run.smithi078:> sudo yum -y install rbd-mirror 2024-09-21T09:20:13.367 INFO:teuthology.orchestra.run.smithi078.stdout:Last metadata expiration check: 0:01:07 ago on Sat 21 Sep 2024 09:19:06 AM UTC. 2024-09-21T09:20:13.501 INFO:teuthology.orchestra.run.smithi078.stdout:Dependencies resolved. 2024-09-21T09:20:13.501 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-21T09:20:13.501 INFO:teuthology.orchestra.run.smithi078.stdout: Package Arch Version Repo Size 2024-09-21T09:20:13.502 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-21T09:20:13.502 INFO:teuthology.orchestra.run.smithi078.stdout:Installing: 2024-09-21T09:20:13.502 INFO:teuthology.orchestra.run.smithi078.stdout: rbd-mirror x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 3.0 M 2024-09-21T09:20:13.502 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:20:13.502 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction Summary 2024-09-21T09:20:13.502 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-21T09:20:13.502 INFO:teuthology.orchestra.run.smithi078.stdout:Install 1 Package 2024-09-21T09:20:13.502 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:20:13.502 INFO:teuthology.orchestra.run.smithi078.stdout:Total download size: 3.0 M 2024-09-21T09:20:13.502 INFO:teuthology.orchestra.run.smithi078.stdout:Installed size: 12 M 2024-09-21T09:20:13.502 INFO:teuthology.orchestra.run.smithi078.stdout:Downloading Packages: 2024-09-21T09:20:13.846 INFO:teuthology.orchestra.run.smithi078.stdout:rbd-mirror-18.2.4-871.g6a1bdf85.el9.x86_64.rpm 8.7 MB/s | 3.0 MB 00:00 2024-09-21T09:20:13.847 INFO:teuthology.orchestra.run.smithi078.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:20:13.847 INFO:teuthology.orchestra.run.smithi078.stdout:Total 8.7 MB/s | 3.0 MB 00:00 2024-09-21T09:20:13.847 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction check 2024-09-21T09:20:13.856 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction check succeeded. 2024-09-21T09:20:13.856 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction test 2024-09-21T09:20:13.929 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction test succeeded. 2024-09-21T09:20:13.930 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction 2024-09-21T09:20:14.342 INFO:teuthology.orchestra.run.smithi078.stdout: Preparing : 1/1 2024-09-21T09:20:14.405 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : rbd-mirror-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:20:14.430 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: rbd-mirror-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:20:14.430 INFO:teuthology.orchestra.run.smithi078.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:20:14.430 INFO:teuthology.orchestra.run.smithi078.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-09-21T09:20:14.430 INFO:teuthology.orchestra.run.smithi078.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-09-21T09:20:14.430 INFO:teuthology.orchestra.run.smithi078.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-09-21T09:20:14.430 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:20:15.521 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : rbd-mirror-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:20:15.522 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:20:15.522 INFO:teuthology.orchestra.run.smithi078.stdout:Installed: 2024-09-21T09:20:15.522 INFO:teuthology.orchestra.run.smithi078.stdout: rbd-mirror-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:20:15.522 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:20:15.522 INFO:teuthology.orchestra.run.smithi078.stdout:Complete! 2024-09-21T09:20:15.734 DEBUG:teuthology.orchestra.run.smithi078:> sudo yum -y install rbd-nbd 2024-09-21T09:20:16.329 INFO:teuthology.orchestra.run.smithi078.stdout:Last metadata expiration check: 0:01:10 ago on Sat 21 Sep 2024 09:19:06 AM UTC. 2024-09-21T09:20:16.462 INFO:teuthology.orchestra.run.smithi078.stdout:Dependencies resolved. 2024-09-21T09:20:16.462 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-21T09:20:16.462 INFO:teuthology.orchestra.run.smithi078.stdout: Package Architecture Version Repository Size 2024-09-21T09:20:16.463 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-21T09:20:16.463 INFO:teuthology.orchestra.run.smithi078.stdout:Installing: 2024-09-21T09:20:16.463 INFO:teuthology.orchestra.run.smithi078.stdout: rbd-nbd x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 172 k 2024-09-21T09:20:16.463 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:20:16.463 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction Summary 2024-09-21T09:20:16.463 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-21T09:20:16.463 INFO:teuthology.orchestra.run.smithi078.stdout:Install 1 Package 2024-09-21T09:20:16.463 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:20:16.464 INFO:teuthology.orchestra.run.smithi078.stdout:Total download size: 172 k 2024-09-21T09:20:16.464 INFO:teuthology.orchestra.run.smithi078.stdout:Installed size: 494 k 2024-09-21T09:20:16.464 INFO:teuthology.orchestra.run.smithi078.stdout:Downloading Packages: 2024-09-21T09:20:16.666 INFO:teuthology.orchestra.run.smithi078.stdout:rbd-nbd-18.2.4-871.g6a1bdf85.el9.x86_64.rpm 851 kB/s | 172 kB 00:00 2024-09-21T09:20:16.666 INFO:teuthology.orchestra.run.smithi078.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:20:16.667 INFO:teuthology.orchestra.run.smithi078.stdout:Total 845 kB/s | 172 kB 00:00 2024-09-21T09:20:16.667 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction check 2024-09-21T09:20:16.674 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction check succeeded. 2024-09-21T09:20:16.674 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction test 2024-09-21T09:20:16.732 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction test succeeded. 2024-09-21T09:20:16.732 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction 2024-09-21T09:20:16.895 INFO:teuthology.orchestra.run.smithi078.stdout: Preparing : 1/1 2024-09-21T09:20:16.978 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : rbd-nbd-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:20:17.391 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: rbd-nbd-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:20:17.648 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : rbd-nbd-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:20:17.648 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:20:17.648 INFO:teuthology.orchestra.run.smithi078.stdout:Installed: 2024-09-21T09:20:17.649 INFO:teuthology.orchestra.run.smithi078.stdout: rbd-nbd-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:20:17.649 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:20:17.649 INFO:teuthology.orchestra.run.smithi078.stdout:Complete! 2024-09-21T09:20:17.884 DEBUG:teuthology.orchestra.run.smithi078:> sudo yum -y install librados2 2024-09-21T09:20:18.479 INFO:teuthology.orchestra.run.smithi078.stdout:Last metadata expiration check: 0:01:12 ago on Sat 21 Sep 2024 09:19:06 AM UTC. 2024-09-21T09:20:18.570 INFO:teuthology.orchestra.run.smithi078.stdout:Package librados2-2:18.2.4-871.g6a1bdf85.el9.x86_64 is already installed. 2024-09-21T09:20:18.612 INFO:teuthology.orchestra.run.smithi078.stdout:Dependencies resolved. 2024-09-21T09:20:18.613 INFO:teuthology.orchestra.run.smithi078.stdout:Nothing to do. 2024-09-21T09:20:18.613 INFO:teuthology.orchestra.run.smithi078.stdout:Complete! 2024-09-21T09:20:18.700 DEBUG:teuthology.parallel:result is None 2024-09-21T09:20:18.701 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-21T09:20:18.701 INFO:teuthology.packaging:ref: None 2024-09-21T09:20:18.701 INFO:teuthology.packaging:tag: None 2024-09-21T09:20:18.701 INFO:teuthology.packaging:branch: reef 2024-09-21T09:20:18.701 INFO:teuthology.packaging:sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:20:18.701 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=reef 2024-09-21T09:20:18.969 DEBUG:teuthology.orchestra.run.smithi078:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-09-21T09:20:18.991 INFO:teuthology.orchestra.run.smithi078.stdout:18.2.4-871.g6a1bdf85.el9 2024-09-21T09:20:18.991 INFO:teuthology.packaging:The installed version of ceph is 18.2.4-871.g6a1bdf85.el9 2024-09-21T09:20:18.991 INFO:teuthology.task.install:The correct ceph version 18.2.4-871.g6a1bdf85 is installed. 2024-09-21T09:20:18.993 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2024-09-21T09:20:18.993 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-21T09:20:18.993 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-09-21T09:20:19.056 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2024-09-21T09:20:19.056 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-21T09:20:19.056 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd of=/usr/bin/daemon-helper 2024-09-21T09:20:19.121 DEBUG:teuthology.orchestra.run.smithi078:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-09-21T09:20:19.187 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2024-09-21T09:20:19.187 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-21T09:20:19.187 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd of=/usr/bin/adjust-ulimits 2024-09-21T09:20:19.251 DEBUG:teuthology.orchestra.run.smithi078:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-09-21T09:20:19.318 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2024-09-21T09:20:19.319 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-21T09:20:19.319 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd of=/usr/bin/stdin-killer 2024-09-21T09:20:19.384 DEBUG:teuthology.orchestra.run.smithi078:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-09-21T09:20:19.450 INFO:teuthology.run_tasks:Running task print... 2024-09-21T09:20:19.459 INFO:teuthology.task.print:**** done installing reef 2024-09-21T09:20:19.459 INFO:teuthology.run_tasks:Running task ceph... 2024-09-21T09:20:19.551 INFO:tasks.ceph:Making ceph log dir writeable by non-root... 2024-09-21T09:20:19.551 DEBUG:teuthology.orchestra.run.smithi078:> sudo chmod 777 /var/log/ceph 2024-09-21T09:20:19.576 INFO:tasks.ceph:Disabling ceph logrotate... 2024-09-21T09:20:19.576 DEBUG:teuthology.orchestra.run.smithi078:> sudo rm -f -- /etc/logrotate.d/ceph 2024-09-21T09:20:19.640 INFO:tasks.ceph:Creating extra log directories... 2024-09-21T09:20:19.641 DEBUG:teuthology.orchestra.run.smithi078:> sudo install -d -m0777 -- /var/log/ceph/valgrind /var/log/ceph/profiling-logger 2024-09-21T09:20:19.718 INFO:tasks.ceph:Creating ceph cluster ceph... 2024-09-21T09:20:19.718 INFO:tasks.ceph:config {'conf': {'global': {'mon warn on pool no app': False, 'ms bind msgr2': False, 'bluestore warn on legacy statfs': False, 'bluestore warn on no per pool omap': False, 'mon pg warn min per osd': 0}, 'mgr': {'client mount timeout': 30, 'debug client': 20, 'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20, 'mon down mkfs grace': 300, 'mon warn on osd down out interval zero': False}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'bitmap', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs', 'mkfs_options': None, 'mount_options': None, 'skip_mgr_daemons': False, 'log_ignorelist': ['overall HEALTH_', '\\(FS_', '\\(MDS_', '\\(OSD_', '\\(MON_DOWN\\)', '\\(CACHE_POOL_', '\\(POOL_', '\\(MGR_DOWN\\)', '\\(PG_', '\\(SMALLER_PGP_NUM\\)', 'Monitor daemon marked osd', 'Behind on trimming', 'Manager daemon', '\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'FS_DEGRADED', 'fs.*is degraded', 'filesystem is degraded', 'FS_INLINE_DATA_DEPRECATED', 'FS_WITH_FAILED_MDS', 'MDS_ALL_DOWN', 'filesystem is offline', 'MDS_DAMAGE', 'MDS_DEGRADED', 'MDS_FAILED', 'MDS_INSUFFICIENT_STANDBY', 'insufficient standby MDS daemons available', 'MDS_UP_LESS_THAN_MAX', 'online, but wants', 'filesystem is online with fewer MDS than max_mds', 'POOL_APP_NOT_ENABLED', 'do not have an application enabled', 'overall HEALTH_', 'Replacing daemon', 'deprecated feature inline_data', 'overall HEALTH_', '\\(OSD_DOWN\\)', '\\(OSD_', 'but it is still running', 'is not responding', 'PG_AVAILABILITY', 'PG_DEGRADED', 'Reduced data availability', 'Degraded data redundancy', 'scrub mismatch', 'ScrubResult', 'wrongly marked', '\\(POOL_APP_NOT_ENABLED\\)', '\\(SLOW_OPS\\)', 'overall HEALTH_', '\\(MON_MSGR2_NOT_ENABLED\\)', 'slow request'], 'cpu_profile': set(), 'cluster': 'ceph', 'mon_bind_msgr2': True, 'mon_bind_addrvec': True} 2024-09-21T09:20:19.719 INFO:tasks.ceph:ctx.config {'archive_path': '/home/teuthworker/archive/teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi/7913639', 'branch': 'main', 'description': 'upgrade/cephfs/nofs/{bluestore-bitmap centos_9.stream conf/{client mds mgr mon osd} no-mds-cluster overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn pg_health} tasks/{0-from/reef 1-upgrade}}', 'email': 'ceph-qa@ceph.com', 'first_in_suite': False, 'job_id': '7913639', 'kernel': {'kdb': 1, 'sha1': 'distro'}, 'last_in_suite': False, 'machine_type': 'smithi', 'meta': [{'desc': 'install ceph/reef latest\n'}], 'name': 'teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi', 'no_nested_subset': False, 'openstack': [{'volumes': {'count': 4, 'size': 10}}], 'os_type': 'centos', 'os_version': '9.stream', 'overrides': {'admin_socket': {'branch': 'main'}, 'ceph': {'cluster-conf': {'client': {'client mount timeout': 600, 'debug client': 20, 'debug ms': 1, 'rados mon op timeout': 900, 'rados osd op timeout': 900}, 'mds': {'debug mds': 20, 'debug mds balancer': 20, 'debug ms': 2, 'mds debug frag': True, 'mds debug scatterstat': True, 'mds op complaint time': 180, 'mds verify scatter': True, 'osd op complaint time': 180, 'rados mon op timeout': 900, 'rados osd op timeout': 900}, 'mon': {'mon op complaint time': 120, 'mon warn on pool no app': False}, 'osd': {'osd op complaint time': 180}}, 'conf': {'global': {'bluestore warn on legacy statfs': False, 'bluestore warn on no per pool omap': False, 'mon pg warn min per osd': 0}, 'mgr': {'client mount timeout': 30, 'debug client': 20, 'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20, 'mon down mkfs grace': 300, 'mon warn on osd down out interval zero': False}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'bitmap', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'FS_DEGRADED', 'fs.*is degraded', 'filesystem is degraded', 'FS_INLINE_DATA_DEPRECATED', 'FS_WITH_FAILED_MDS', 'MDS_ALL_DOWN', 'filesystem is offline', 'MDS_DAMAGE', 'MDS_DEGRADED', 'MDS_FAILED', 'MDS_INSUFFICIENT_STANDBY', 'insufficient standby MDS daemons available', 'MDS_UP_LESS_THAN_MAX', 'online, but wants', 'filesystem is online with fewer MDS than max_mds', 'POOL_APP_NOT_ENABLED', 'do not have an application enabled', 'overall HEALTH_', 'Replacing daemon', 'deprecated feature inline_data', 'overall HEALTH_', '\\(OSD_DOWN\\)', '\\(OSD_', 'but it is still running', 'is not responding', 'PG_AVAILABILITY', 'PG_DEGRADED', 'Reduced data availability', 'Degraded data redundancy', 'scrub mismatch', 'ScrubResult', 'wrongly marked', '\\(POOL_APP_NOT_ENABLED\\)', '\\(SLOW_OPS\\)', 'overall HEALTH_', '\\(MON_MSGR2_NOT_ENABLED\\)', 'slow request'], 'sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208'}, 'ceph-deploy': {'bluestore': True, 'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs'}, 'install': {'ceph': {'flavor': 'default', 'sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208'}}, 'selinux': {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0']}, 'thrashosds': {'bdev_inject_crash': 2, 'bdev_inject_crash_probability': 0.5}, 'workunit': {'branch': 'main', 'sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208'}}, 'owner': 'scheduled_teuthology@teuthology', 'priority': 850, 'repo': 'https://git.ceph.com/ceph.git', 'roles': [['mon.a', 'mon.b', 'mon.c', 'mgr.x', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3']], 'seed': 5357, 'sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208', 'sleep_before_teardown': 0, 'subset': '31/32', 'suite': 'upgrade', 'suite_branch': 'main', 'suite_path': '/home/teuthworker/src/git.ceph.com_ceph_62da54b4b83a96ca9976b7e1d0c8272a564a2208/qa', 'suite_relpath': 'qa', 'suite_repo': 'https://git.ceph.com/ceph.git', 'suite_sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208', 'targets': {'smithi078.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBOL5fG+Lq2YIfK3G2HmCT3mD3yizsUrqLk2UWnY/YmsJijyt8gwZ2Waqg5sm3mvb51Gx3FcxoB6m0JboN8aTUNw='}, 'tasks': [{'internal.check_packages': None}, {'internal.buildpackages_prep': None}, {'internal.save_config': None}, {'internal.check_lock': None}, {'internal.add_remotes': None}, {'console_log': None}, {'internal.connect': None}, {'internal.push_inventory': None}, {'internal.serialize_remote_roles': None}, {'internal.check_conflict': None}, {'internal.check_ceph_data': None}, {'internal.vm_setup': None}, {'kernel': {'kdb': 1, 'sha1': 'distro'}}, {'internal.base': None}, {'internal.archive_upload': None}, {'internal.archive': None}, {'internal.coredump': None}, {'internal.sudo': None}, {'internal.syslog': None}, {'internal.timer': None}, {'pcp': None}, {'selinux': None}, {'ansible.cephlab': None}, {'clock': None}, {'install': {'branch': 'reef', 'exclude_packages': ['librados3', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'cephadm', 'ceph-volume'], 'extra_packages': ['librados2'], 'flavor': 'default', 'sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208'}}, {'print': '**** done installing reef'}, {'ceph': {'conf': {'global': {'mon warn on pool no app': False, 'ms bind msgr2': False, 'bluestore warn on legacy statfs': False, 'bluestore warn on no per pool omap': False, 'mon pg warn min per osd': 0}, 'mgr': {'client mount timeout': 30, 'debug client': 20, 'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20, 'mon down mkfs grace': 300, 'mon warn on osd down out interval zero': False}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'bitmap', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'log-ignorelist': ['overall HEALTH_', '\\(FS_', '\\(MDS_', '\\(OSD_', '\\(MON_DOWN\\)', '\\(CACHE_POOL_', '\\(POOL_', '\\(MGR_DOWN\\)', '\\(PG_', '\\(SMALLER_PGP_NUM\\)', 'Monitor daemon marked osd', 'Behind on trimming', 'Manager daemon', '\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'FS_DEGRADED', 'fs.*is degraded', 'filesystem is degraded', 'FS_INLINE_DATA_DEPRECATED', 'FS_WITH_FAILED_MDS', 'MDS_ALL_DOWN', 'filesystem is offline', 'MDS_DAMAGE', 'MDS_DEGRADED', 'MDS_FAILED', 'MDS_INSUFFICIENT_STANDBY', 'insufficient standby MDS daemons available', 'MDS_UP_LESS_THAN_MAX', 'online, but wants', 'filesystem is online with fewer MDS than max_mds', 'POOL_APP_NOT_ENABLED', 'do not have an application enabled', 'overall HEALTH_', 'Replacing daemon', 'deprecated feature inline_data', 'overall HEALTH_', '\\(OSD_DOWN\\)', '\\(OSD_', 'but it is still running', 'is not responding', 'PG_AVAILABILITY', 'PG_DEGRADED', 'Reduced data availability', 'Degraded data redundancy', 'scrub mismatch', 'ScrubResult', 'wrongly marked', '\\(POOL_APP_NOT_ENABLED\\)', '\\(SLOW_OPS\\)', 'overall HEALTH_', '\\(MON_MSGR2_NOT_ENABLED\\)', 'slow request'], 'cluster-conf': {'client': {'client mount timeout': 600, 'debug client': 20, 'debug ms': 1, 'rados mon op timeout': 900, 'rados osd op timeout': 900}, 'mds': {'debug mds': 20, 'debug mds balancer': 20, 'debug ms': 2, 'mds debug frag': True, 'mds debug scatterstat': True, 'mds op complaint time': 180, 'mds verify scatter': True, 'osd op complaint time': 180, 'rados mon op timeout': 900, 'rados osd op timeout': 900}, 'mon': {'mon op complaint time': 120, 'mon warn on pool no app': False}, 'osd': {'osd op complaint time': 180}}, 'flavor': 'default', 'fs': 'xfs', 'sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208', 'cluster': 'ceph'}}, {'exec': {'osd.0': ['ceph osd set-require-min-compat-client reef']}}, {'print': '**** done ceph'}, {'print': '*** upgrading, no cephfs present'}, {'exec': {'mon.a': ['ceph fs dump']}}, {'install.upgrade': {'mon.a': None}}, {'print': '**** done install.upgrade'}, {'ceph.restart': {'daemons': ['mon.*', 'mgr.*'], 'mon-health-to-clog': False, 'wait-for-healthy': False}}, {'ceph.healthy': None}, {'ceph.restart': {'daemons': ['osd.*'], 'wait-for-healthy': False, 'wait-for-osds-up': True}}, {'exec': {'mon.a': ['ceph versions', 'ceph osd dump -f json-pretty', 'ceph fs dump', 'ceph osd require-osd-release squid', 'for f in `ceph osd pool ls` ; do ceph osd pool set $f pg_autoscale_mode off ; done']}}, {'ceph.healthy': None}, {'print': '**** done ceph.restart'}], 'teuthology': {'fragments_dropped': [], 'meta': {}, 'postmerge': []}, 'teuthology_branch': 'main', 'teuthology_sha1': '3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a', 'timestamp': '2024-09-21_03:08:26', 'tube': 'smithi', 'user': 'teuthology', 'verbose': False, 'worker_log': '/home/teuthworker/archive/worker_logs/dispatcher.smithi.2902'} 2024-09-21T09:20:19.719 DEBUG:teuthology.orchestra.run.smithi078:> install -d -m0755 -- /home/ubuntu/cephtest/ceph.data 2024-09-21T09:20:19.761 DEBUG:teuthology.orchestra.run.smithi078:> sudo install -d -m0777 -- /var/run/ceph 2024-09-21T09:20:19.826 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-21T09:20:19.826 DEBUG:teuthology.orchestra.run.smithi078:> dd if=/scratch_devs of=/dev/stdout 2024-09-21T09:20:19.881 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-21T09:20:19.881 DEBUG:teuthology.orchestra.run.smithi078:> stat /dev/vg_nvme/lv_1 2024-09-21T09:20:19.936 INFO:teuthology.orchestra.run.smithi078.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-21T09:20:19.936 INFO:teuthology.orchestra.run.smithi078.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T09:20:19.936 INFO:teuthology.orchestra.run.smithi078.stdout:Device: 5h/5d Inode: 665 Links: 1 2024-09-21T09:20:19.936 INFO:teuthology.orchestra.run.smithi078.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T09:20:19.936 INFO:teuthology.orchestra.run.smithi078.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T09:20:19.936 INFO:teuthology.orchestra.run.smithi078.stdout:Access: 2024-09-21 09:20:14.690810916 +0000 2024-09-21T09:20:19.936 INFO:teuthology.orchestra.run.smithi078.stdout:Modify: 2024-09-21 09:18:30.613675646 +0000 2024-09-21T09:20:19.936 INFO:teuthology.orchestra.run.smithi078.stdout:Change: 2024-09-21 09:18:30.613675646 +0000 2024-09-21T09:20:19.936 INFO:teuthology.orchestra.run.smithi078.stdout: Birth: 2024-09-21 09:18:30.613675646 +0000 2024-09-21T09:20:19.937 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-21T09:20:19.998 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records in 2024-09-21T09:20:19.999 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records out 2024-09-21T09:20:19.999 INFO:teuthology.orchestra.run.smithi078.stderr:512 bytes copied, 0.000184443 s, 2.8 MB/s 2024-09-21T09:20:20.000 DEBUG:teuthology.orchestra.run.smithi078:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-21T09:20:20.057 DEBUG:teuthology.orchestra.run.smithi078:> stat /dev/vg_nvme/lv_2 2024-09-21T09:20:20.112 INFO:teuthology.orchestra.run.smithi078.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-21T09:20:20.112 INFO:teuthology.orchestra.run.smithi078.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T09:20:20.112 INFO:teuthology.orchestra.run.smithi078.stdout:Device: 5h/5d Inode: 687 Links: 1 2024-09-21T09:20:20.112 INFO:teuthology.orchestra.run.smithi078.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T09:20:20.112 INFO:teuthology.orchestra.run.smithi078.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T09:20:20.112 INFO:teuthology.orchestra.run.smithi078.stdout:Access: 2024-09-21 09:20:14.690810916 +0000 2024-09-21T09:20:20.112 INFO:teuthology.orchestra.run.smithi078.stdout:Modify: 2024-09-21 09:18:30.978670251 +0000 2024-09-21T09:20:20.112 INFO:teuthology.orchestra.run.smithi078.stdout:Change: 2024-09-21 09:18:30.978670251 +0000 2024-09-21T09:20:20.112 INFO:teuthology.orchestra.run.smithi078.stdout: Birth: 2024-09-21 09:18:30.978670251 +0000 2024-09-21T09:20:20.113 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-21T09:20:20.174 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records in 2024-09-21T09:20:20.175 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records out 2024-09-21T09:20:20.175 INFO:teuthology.orchestra.run.smithi078.stderr:512 bytes copied, 0.000203053 s, 2.5 MB/s 2024-09-21T09:20:20.176 DEBUG:teuthology.orchestra.run.smithi078:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-21T09:20:20.231 DEBUG:teuthology.orchestra.run.smithi078:> stat /dev/vg_nvme/lv_3 2024-09-21T09:20:20.288 INFO:teuthology.orchestra.run.smithi078.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-21T09:20:20.288 INFO:teuthology.orchestra.run.smithi078.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T09:20:20.288 INFO:teuthology.orchestra.run.smithi078.stdout:Device: 5h/5d Inode: 704 Links: 1 2024-09-21T09:20:20.288 INFO:teuthology.orchestra.run.smithi078.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T09:20:20.289 INFO:teuthology.orchestra.run.smithi078.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T09:20:20.289 INFO:teuthology.orchestra.run.smithi078.stdout:Access: 2024-09-21 09:20:14.690810916 +0000 2024-09-21T09:20:20.289 INFO:teuthology.orchestra.run.smithi078.stdout:Modify: 2024-09-21 09:18:31.320665197 +0000 2024-09-21T09:20:20.289 INFO:teuthology.orchestra.run.smithi078.stdout:Change: 2024-09-21 09:18:31.320665197 +0000 2024-09-21T09:20:20.289 INFO:teuthology.orchestra.run.smithi078.stdout: Birth: 2024-09-21 09:18:31.320665197 +0000 2024-09-21T09:20:20.289 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-21T09:20:20.351 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records in 2024-09-21T09:20:20.351 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records out 2024-09-21T09:20:20.351 INFO:teuthology.orchestra.run.smithi078.stderr:512 bytes copied, 0.000221427 s, 2.3 MB/s 2024-09-21T09:20:20.353 DEBUG:teuthology.orchestra.run.smithi078:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-21T09:20:20.410 DEBUG:teuthology.orchestra.run.smithi078:> stat /dev/vg_nvme/lv_4 2024-09-21T09:20:20.465 INFO:teuthology.orchestra.run.smithi078.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-21T09:20:20.465 INFO:teuthology.orchestra.run.smithi078.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T09:20:20.465 INFO:teuthology.orchestra.run.smithi078.stdout:Device: 5h/5d Inode: 723 Links: 1 2024-09-21T09:20:20.465 INFO:teuthology.orchestra.run.smithi078.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T09:20:20.466 INFO:teuthology.orchestra.run.smithi078.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T09:20:20.466 INFO:teuthology.orchestra.run.smithi078.stdout:Access: 2024-09-21 09:20:14.691810901 +0000 2024-09-21T09:20:20.466 INFO:teuthology.orchestra.run.smithi078.stdout:Modify: 2024-09-21 09:18:31.661660157 +0000 2024-09-21T09:20:20.466 INFO:teuthology.orchestra.run.smithi078.stdout:Change: 2024-09-21 09:18:31.661660157 +0000 2024-09-21T09:20:20.466 INFO:teuthology.orchestra.run.smithi078.stdout: Birth: 2024-09-21 09:18:31.661660157 +0000 2024-09-21T09:20:20.466 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-21T09:20:20.528 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records in 2024-09-21T09:20:20.528 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records out 2024-09-21T09:20:20.528 INFO:teuthology.orchestra.run.smithi078.stderr:512 bytes copied, 0.000189613 s, 2.7 MB/s 2024-09-21T09:20:20.529 DEBUG:teuthology.orchestra.run.smithi078:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-21T09:20:20.586 INFO:tasks.ceph:osd dev map: {'osd.0': '/dev/vg_nvme/lv_1', 'osd.1': '/dev/vg_nvme/lv_2', 'osd.2': '/dev/vg_nvme/lv_3', 'osd.3': '/dev/vg_nvme/lv_4'} 2024-09-21T09:20:20.586 INFO:tasks.ceph:remote_to_roles_to_devs: {Remote(name='ubuntu@smithi078.front.sepia.ceph.com'): {'osd.0': '/dev/vg_nvme/lv_1', 'osd.1': '/dev/vg_nvme/lv_2', 'osd.2': '/dev/vg_nvme/lv_3', 'osd.3': '/dev/vg_nvme/lv_4'}} 2024-09-21T09:20:20.586 INFO:tasks.ceph:Generating config... 2024-09-21T09:20:20.587 INFO:tasks.ceph:[global] mon warn on pool no app = False 2024-09-21T09:20:20.587 INFO:tasks.ceph:[global] ms bind msgr2 = False 2024-09-21T09:20:20.587 INFO:tasks.ceph:[global] bluestore warn on legacy statfs = False 2024-09-21T09:20:20.587 INFO:tasks.ceph:[global] bluestore warn on no per pool omap = False 2024-09-21T09:20:20.587 INFO:tasks.ceph:[global] mon pg warn min per osd = 0 2024-09-21T09:20:20.588 INFO:tasks.ceph:[mgr] client mount timeout = 30 2024-09-21T09:20:20.588 INFO:tasks.ceph:[mgr] debug client = 20 2024-09-21T09:20:20.588 INFO:tasks.ceph:[mgr] debug mgr = 20 2024-09-21T09:20:20.588 INFO:tasks.ceph:[mgr] debug ms = 1 2024-09-21T09:20:20.588 INFO:tasks.ceph:[mon] debug mon = 20 2024-09-21T09:20:20.588 INFO:tasks.ceph:[mon] debug ms = 1 2024-09-21T09:20:20.588 INFO:tasks.ceph:[mon] debug paxos = 20 2024-09-21T09:20:20.588 INFO:tasks.ceph:[mon] mon down mkfs grace = 300 2024-09-21T09:20:20.588 INFO:tasks.ceph:[mon] mon warn on osd down out interval zero = False 2024-09-21T09:20:20.588 INFO:tasks.ceph:[osd] bdev async discard = True 2024-09-21T09:20:20.588 INFO:tasks.ceph:[osd] bdev enable discard = True 2024-09-21T09:20:20.588 INFO:tasks.ceph:[osd] bluestore allocator = bitmap 2024-09-21T09:20:20.588 INFO:tasks.ceph:[osd] bluestore block size = 96636764160 2024-09-21T09:20:20.588 INFO:tasks.ceph:[osd] bluestore fsck on mount = True 2024-09-21T09:20:20.589 INFO:tasks.ceph:[osd] debug bluefs = 1/20 2024-09-21T09:20:20.589 INFO:tasks.ceph:[osd] debug bluestore = 1/20 2024-09-21T09:20:20.589 INFO:tasks.ceph:[osd] debug ms = 1 2024-09-21T09:20:20.589 INFO:tasks.ceph:[osd] debug osd = 20 2024-09-21T09:20:20.589 INFO:tasks.ceph:[osd] debug rocksdb = 4/10 2024-09-21T09:20:20.589 INFO:tasks.ceph:[osd] mon osd backfillfull_ratio = 0.85 2024-09-21T09:20:20.589 INFO:tasks.ceph:[osd] mon osd full ratio = 0.9 2024-09-21T09:20:20.589 INFO:tasks.ceph:[osd] mon osd nearfull ratio = 0.8 2024-09-21T09:20:20.589 INFO:tasks.ceph:[osd] osd failsafe full ratio = 0.95 2024-09-21T09:20:20.589 INFO:tasks.ceph:[osd] osd objectstore = bluestore 2024-09-21T09:20:20.589 INFO:tasks.ceph:Setting up mon.a... 2024-09-21T09:20:20.590 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring /etc/ceph/ceph.keyring 2024-09-21T09:20:20.696 INFO:teuthology.orchestra.run.smithi078.stdout:creating /etc/ceph/ceph.keyring 2024-09-21T09:20:20.699 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=mon. /etc/ceph/ceph.keyring 2024-09-21T09:20:20.740 DEBUG:teuthology.orchestra.run.smithi078:> sudo chmod 0644 /etc/ceph/ceph.keyring 2024-09-21T09:20:20.804 DEBUG:tasks.ceph:Ceph mon addresses: [('mon.a', '172.21.15.78'), ('mon.b', '[v2:172.21.15.78:3301,v1:172.21.15.78:6790]'), ('mon.c', '[v2:172.21.15.78:3302,v1:172.21.15.78:6791]')] 2024-09-21T09:20:20.805 DEBUG:tasks.ceph:writing out conf {'global': {'chdir': '', 'pid file': '/var/run/ceph/$cluster-$name.pid', 'auth supported': 'cephx', 'filestore xattr use omap': 'true', 'mon clock drift allowed': '1.000', 'osd crush chooseleaf type': '0', 'auth debug': 'true', 'ms die on old message': 'true', 'ms die on bug': 'true', 'mon max pg per osd': '10000', 'mon pg warn max object skew': '0', 'osd_pool_default_pg_autoscale_mode': 'off', 'osd pool default size': '2', 'mon osd allow primary affinity': 'true', 'mon osd allow pg remap': 'true', 'mon warn on legacy crush tunables': 'false', 'mon warn on crush straw calc version zero': 'false', 'mon warn on no sortbitwise': 'false', 'mon warn on osd down out interval zero': 'false', 'mon warn on too few osds': 'false', 'mon_warn_on_pool_pg_num_not_power_of_two': 'false', 'mon_warn_on_pool_no_redundancy': 'false', 'mon_allow_pool_size_one': 'true', 'osd pool default erasure code profile': 'plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd', 'osd default data pool replay window': '5', 'mon allow pool delete': 'true', 'mon cluster log file level': 'debug', 'debug asserts on shutdown': 'true', 'mon health detail to clog': 'false', 'mon host': '172.21.15.78,[v2:172.21.15.78:3301,v1:172.21.15.78:6790],[v2:172.21.15.78:3302,v1:172.21.15.78:6791]', 'mon warn on pool no app': False, 'ms bind msgr2': False, 'bluestore warn on legacy statfs': False, 'bluestore warn on no per pool omap': False, 'mon pg warn min per osd': 0}, 'osd': {'osd journal size': '100', 'osd scrub load threshold': '5.0', 'osd scrub max interval': '600', 'osd mclock profile': 'high_recovery_ops', 'osd recover clone overlap': 'true', 'osd recovery max chunk': '1048576', 'osd debug shutdown': 'true', 'osd debug op order': 'true', 'osd debug verify stray on activate': 'true', 'osd debug trim objects': 'true', 'osd open classes on start': 'true', 'osd debug pg log writeout': 'true', 'osd deep scrub update digest min age': '30', 'osd map max advance': '10', 'journal zero on create': 'true', 'filestore ondisk finisher threads': '3', 'filestore apply finisher threads': '3', 'bdev debug aio': 'true', 'osd debug misdirected ops': 'true', 'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'bitmap', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}, 'mgr': {'debug ms': 1, 'debug mgr': 20, 'debug mon': '20', 'debug auth': '20', 'mon reweight min pgs per osd': '4', 'mon reweight min bytes per osd': '10', 'mgr/telemetry/nag': 'false', 'client mount timeout': 30, 'debug client': 20}, 'mon': {'debug ms': 1, 'debug mon': 20, 'debug paxos': 20, 'debug auth': '20', 'mon data avail warn': '5', 'mon mgr mkfs grace': '240', 'mon reweight min pgs per osd': '4', 'mon osd reporter subtree level': 'osd', 'mon osd prime pg temp': 'true', 'mon reweight min bytes per osd': '10', 'auth mon ticket ttl': '660', 'auth service ticket ttl': '240', 'mon_warn_on_insecure_global_id_reclaim': 'false', 'mon_warn_on_insecure_global_id_reclaim_allowed': 'false', 'mon_down_mkfs_grace': '2m', 'mon_warn_on_filestore_osds': 'false', 'mon down mkfs grace': 300, 'mon warn on osd down out interval zero': False}, 'client': {'rgw cache enabled': 'true', 'rgw enable ops log': 'true', 'rgw enable usage log': 'true', 'log file': '/var/log/ceph/$cluster-$name.$pid.log', 'admin socket': '/var/run/ceph/$cluster-$name.$pid.asok'}, 'mon.a': {}, 'mon.b': {}, 'mon.c': {}} 2024-09-21T09:20:20.805 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-21T09:20:20.806 DEBUG:teuthology.orchestra.run.smithi078:> dd of=/home/ubuntu/cephtest/ceph.tmp.conf 2024-09-21T09:20:20.859 DEBUG:teuthology.orchestra.run.smithi078:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage monmaptool -c /home/ubuntu/cephtest/ceph.tmp.conf --create --clobber --enable-all-features --add a 172.21.15.78 --addv b '[v2:172.21.15.78:3301,v1:172.21.15.78:6790]' --addv c '[v2:172.21.15.78:3302,v1:172.21.15.78:6791]' --print /home/ubuntu/cephtest/ceph.monmap 2024-09-21T09:20:20.944 INFO:teuthology.orchestra.run.smithi078.stdout:monmaptool: monmap file /home/ubuntu/cephtest/ceph.monmap 2024-09-21T09:20:20.944 INFO:teuthology.orchestra.run.smithi078.stdout:monmaptool: generated fsid 3633ad52-1063-4406-97d9-d53c339f43ca 2024-09-21T09:20:20.944 INFO:teuthology.orchestra.run.smithi078.stdout:setting min_mon_release = pacific 2024-09-21T09:20:20.944 INFO:teuthology.orchestra.run.smithi078.stdout:epoch 0 2024-09-21T09:20:20.944 INFO:teuthology.orchestra.run.smithi078.stdout:fsid 3633ad52-1063-4406-97d9-d53c339f43ca 2024-09-21T09:20:20.944 INFO:teuthology.orchestra.run.smithi078.stdout:last_changed 2024-09-21T09:20:20.943512+0000 2024-09-21T09:20:20.945 INFO:teuthology.orchestra.run.smithi078.stdout:created 2024-09-21T09:20:20.943512+0000 2024-09-21T09:20:20.945 INFO:teuthology.orchestra.run.smithi078.stdout:min_mon_release 16 (pacific) 2024-09-21T09:20:20.945 INFO:teuthology.orchestra.run.smithi078.stdout:election_strategy: 1 2024-09-21T09:20:20.945 INFO:teuthology.orchestra.run.smithi078.stdout:0: [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] mon.a 2024-09-21T09:20:20.945 INFO:teuthology.orchestra.run.smithi078.stdout:1: [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] mon.b 2024-09-21T09:20:20.945 INFO:teuthology.orchestra.run.smithi078.stdout:2: [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] mon.c 2024-09-21T09:20:20.945 INFO:teuthology.orchestra.run.smithi078.stdout:monmaptool: writing epoch 0 to /home/ubuntu/cephtest/ceph.monmap (3 monitors) 2024-09-21T09:20:20.946 DEBUG:teuthology.orchestra.run.smithi078:> rm -- /home/ubuntu/cephtest/ceph.tmp.conf 2024-09-21T09:20:20.961 INFO:tasks.ceph:Writing /etc/ceph/ceph.conf for FSID 3633ad52-1063-4406-97d9-d53c339f43ca... 2024-09-21T09:20:20.962 DEBUG:teuthology.orchestra.run.smithi078:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/ceph.conf && sudo chmod 0644 /etc/ceph/ceph.conf > /dev/null 2024-09-21T09:20:21.043 INFO:teuthology.orchestra.run.smithi078.stdout:[global] 2024-09-21T09:20:21.043 INFO:teuthology.orchestra.run.smithi078.stdout: chdir = "" 2024-09-21T09:20:21.044 INFO:teuthology.orchestra.run.smithi078.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2024-09-21T09:20:21.044 INFO:teuthology.orchestra.run.smithi078.stdout: auth supported = cephx 2024-09-21T09:20:21.044 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:20:21.044 INFO:teuthology.orchestra.run.smithi078.stdout: filestore xattr use omap = true 2024-09-21T09:20:21.044 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:20:21.044 INFO:teuthology.orchestra.run.smithi078.stdout: mon clock drift allowed = 1.000 2024-09-21T09:20:21.044 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:20:21.044 INFO:teuthology.orchestra.run.smithi078.stdout: osd crush chooseleaf type = 0 2024-09-21T09:20:21.044 INFO:teuthology.orchestra.run.smithi078.stdout: auth debug = true 2024-09-21T09:20:21.044 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:20:21.044 INFO:teuthology.orchestra.run.smithi078.stdout: ms die on old message = true 2024-09-21T09:20:21.044 INFO:teuthology.orchestra.run.smithi078.stdout: ms die on bug = true 2024-09-21T09:20:21.044 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:20:21.044 INFO:teuthology.orchestra.run.smithi078.stdout: mon max pg per osd = 10000 # >= luminous 2024-09-21T09:20:21.045 INFO:teuthology.orchestra.run.smithi078.stdout: mon pg warn max object skew = 0 2024-09-21T09:20:21.045 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:20:21.045 INFO:teuthology.orchestra.run.smithi078.stdout: # disable pg_autoscaler by default for new pools 2024-09-21T09:20:21.045 INFO:teuthology.orchestra.run.smithi078.stdout: osd_pool_default_pg_autoscale_mode = off 2024-09-21T09:20:21.045 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:20:21.045 INFO:teuthology.orchestra.run.smithi078.stdout: osd pool default size = 2 2024-09-21T09:20:21.045 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:20:21.045 INFO:teuthology.orchestra.run.smithi078.stdout: mon osd allow primary affinity = true 2024-09-21T09:20:21.045 INFO:teuthology.orchestra.run.smithi078.stdout: mon osd allow pg remap = true 2024-09-21T09:20:21.045 INFO:teuthology.orchestra.run.smithi078.stdout: mon warn on legacy crush tunables = false 2024-09-21T09:20:21.045 INFO:teuthology.orchestra.run.smithi078.stdout: mon warn on crush straw calc version zero = false 2024-09-21T09:20:21.045 INFO:teuthology.orchestra.run.smithi078.stdout: mon warn on no sortbitwise = false 2024-09-21T09:20:21.045 INFO:teuthology.orchestra.run.smithi078.stdout: mon warn on osd down out interval zero = false 2024-09-21T09:20:21.046 INFO:teuthology.orchestra.run.smithi078.stdout: mon warn on too few osds = false 2024-09-21T09:20:21.046 INFO:teuthology.orchestra.run.smithi078.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2024-09-21T09:20:21.046 INFO:teuthology.orchestra.run.smithi078.stdout: mon_warn_on_pool_no_redundancy = false 2024-09-21T09:20:21.046 INFO:teuthology.orchestra.run.smithi078.stdout: mon_allow_pool_size_one = true 2024-09-21T09:20:21.046 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:20:21.046 INFO:teuthology.orchestra.run.smithi078.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2024-09-21T09:20:21.046 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:20:21.046 INFO:teuthology.orchestra.run.smithi078.stdout: osd default data pool replay window = 5 2024-09-21T09:20:21.046 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:20:21.046 INFO:teuthology.orchestra.run.smithi078.stdout: mon allow pool delete = true 2024-09-21T09:20:21.046 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:20:21.046 INFO:teuthology.orchestra.run.smithi078.stdout: mon cluster log file level = debug 2024-09-21T09:20:21.046 INFO:teuthology.orchestra.run.smithi078.stdout: debug asserts on shutdown = true 2024-09-21T09:20:21.046 INFO:teuthology.orchestra.run.smithi078.stdout: mon health detail to clog = false 2024-09-21T09:20:21.047 INFO:teuthology.orchestra.run.smithi078.stdout: mon host = "172.21.15.78,[v2:172.21.15.78:3301,v1:172.21.15.78:6790],[v2:172.21.15.78:3302,v1:172.21.15.78:6791]" 2024-09-21T09:20:21.047 INFO:teuthology.orchestra.run.smithi078.stdout: mon warn on pool no app = False 2024-09-21T09:20:21.047 INFO:teuthology.orchestra.run.smithi078.stdout: ms bind msgr2 = False 2024-09-21T09:20:21.047 INFO:teuthology.orchestra.run.smithi078.stdout: bluestore warn on legacy statfs = False 2024-09-21T09:20:21.047 INFO:teuthology.orchestra.run.smithi078.stdout: bluestore warn on no per pool omap = False 2024-09-21T09:20:21.047 INFO:teuthology.orchestra.run.smithi078.stdout: mon pg warn min per osd = 0 2024-09-21T09:20:21.047 INFO:teuthology.orchestra.run.smithi078.stdout: fsid = 3633ad52-1063-4406-97d9-d53c339f43ca 2024-09-21T09:20:21.047 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:20:21.047 INFO:teuthology.orchestra.run.smithi078.stdout:[osd] 2024-09-21T09:20:21.047 INFO:teuthology.orchestra.run.smithi078.stdout: osd journal size = 100 2024-09-21T09:20:21.047 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:20:21.047 INFO:teuthology.orchestra.run.smithi078.stdout: osd scrub load threshold = 5.0 2024-09-21T09:20:21.047 INFO:teuthology.orchestra.run.smithi078.stdout: osd scrub max interval = 600 2024-09-21T09:20:21.047 INFO:teuthology.orchestra.run.smithi078.stdout: osd mclock profile = high_recovery_ops 2024-09-21T09:20:21.048 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:20:21.048 INFO:teuthology.orchestra.run.smithi078.stdout: osd recover clone overlap = true 2024-09-21T09:20:21.048 INFO:teuthology.orchestra.run.smithi078.stdout: osd recovery max chunk = 1048576 2024-09-21T09:20:21.048 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:20:21.048 INFO:teuthology.orchestra.run.smithi078.stdout: osd debug shutdown = true 2024-09-21T09:20:21.048 INFO:teuthology.orchestra.run.smithi078.stdout: osd debug op order = true 2024-09-21T09:20:21.048 INFO:teuthology.orchestra.run.smithi078.stdout: osd debug verify stray on activate = true 2024-09-21T09:20:21.048 INFO:teuthology.orchestra.run.smithi078.stdout: osd debug trim objects = true 2024-09-21T09:20:21.048 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:20:21.048 INFO:teuthology.orchestra.run.smithi078.stdout: osd open classes on start = true 2024-09-21T09:20:21.048 INFO:teuthology.orchestra.run.smithi078.stdout: osd debug pg log writeout = true 2024-09-21T09:20:21.048 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:20:21.048 INFO:teuthology.orchestra.run.smithi078.stdout: osd deep scrub update digest min age = 30 2024-09-21T09:20:21.048 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:20:21.049 INFO:teuthology.orchestra.run.smithi078.stdout: osd map max advance = 10 2024-09-21T09:20:21.049 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:20:21.049 INFO:teuthology.orchestra.run.smithi078.stdout: journal zero on create = true 2024-09-21T09:20:21.049 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:20:21.049 INFO:teuthology.orchestra.run.smithi078.stdout: filestore ondisk finisher threads = 3 2024-09-21T09:20:21.049 INFO:teuthology.orchestra.run.smithi078.stdout: filestore apply finisher threads = 3 2024-09-21T09:20:21.049 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:20:21.049 INFO:teuthology.orchestra.run.smithi078.stdout: bdev debug aio = true 2024-09-21T09:20:21.049 INFO:teuthology.orchestra.run.smithi078.stdout: osd debug misdirected ops = true 2024-09-21T09:20:21.049 INFO:teuthology.orchestra.run.smithi078.stdout: bdev async discard = True 2024-09-21T09:20:21.049 INFO:teuthology.orchestra.run.smithi078.stdout: bdev enable discard = True 2024-09-21T09:20:21.049 INFO:teuthology.orchestra.run.smithi078.stdout: bluestore allocator = bitmap 2024-09-21T09:20:21.049 INFO:teuthology.orchestra.run.smithi078.stdout: bluestore block size = 96636764160 2024-09-21T09:20:21.049 INFO:teuthology.orchestra.run.smithi078.stdout: bluestore fsck on mount = True 2024-09-21T09:20:21.049 INFO:teuthology.orchestra.run.smithi078.stdout: debug bluefs = 1/20 2024-09-21T09:20:21.050 INFO:teuthology.orchestra.run.smithi078.stdout: debug bluestore = 1/20 2024-09-21T09:20:21.050 INFO:teuthology.orchestra.run.smithi078.stdout: debug ms = 1 2024-09-21T09:20:21.050 INFO:teuthology.orchestra.run.smithi078.stdout: debug osd = 20 2024-09-21T09:20:21.050 INFO:teuthology.orchestra.run.smithi078.stdout: debug rocksdb = 4/10 2024-09-21T09:20:21.050 INFO:teuthology.orchestra.run.smithi078.stdout: mon osd backfillfull_ratio = 0.85 2024-09-21T09:20:21.050 INFO:teuthology.orchestra.run.smithi078.stdout: mon osd full ratio = 0.9 2024-09-21T09:20:21.050 INFO:teuthology.orchestra.run.smithi078.stdout: mon osd nearfull ratio = 0.8 2024-09-21T09:20:21.050 INFO:teuthology.orchestra.run.smithi078.stdout: osd failsafe full ratio = 0.95 2024-09-21T09:20:21.050 INFO:teuthology.orchestra.run.smithi078.stdout: osd objectstore = bluestore 2024-09-21T09:20:21.050 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:20:21.050 INFO:teuthology.orchestra.run.smithi078.stdout:[mgr] 2024-09-21T09:20:21.050 INFO:teuthology.orchestra.run.smithi078.stdout: debug ms = 1 2024-09-21T09:20:21.050 INFO:teuthology.orchestra.run.smithi078.stdout: debug mgr = 20 2024-09-21T09:20:21.050 INFO:teuthology.orchestra.run.smithi078.stdout: debug mon = 20 2024-09-21T09:20:21.051 INFO:teuthology.orchestra.run.smithi078.stdout: debug auth = 20 2024-09-21T09:20:21.051 INFO:teuthology.orchestra.run.smithi078.stdout: mon reweight min pgs per osd = 4 2024-09-21T09:20:21.051 INFO:teuthology.orchestra.run.smithi078.stdout: mon reweight min bytes per osd = 10 2024-09-21T09:20:21.051 INFO:teuthology.orchestra.run.smithi078.stdout: mgr/telemetry/nag = false 2024-09-21T09:20:21.051 INFO:teuthology.orchestra.run.smithi078.stdout: client mount timeout = 30 2024-09-21T09:20:21.051 INFO:teuthology.orchestra.run.smithi078.stdout: debug client = 20 2024-09-21T09:20:21.051 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:20:21.051 INFO:teuthology.orchestra.run.smithi078.stdout:[mon] 2024-09-21T09:20:21.051 INFO:teuthology.orchestra.run.smithi078.stdout: debug ms = 1 2024-09-21T09:20:21.051 INFO:teuthology.orchestra.run.smithi078.stdout: debug mon = 20 2024-09-21T09:20:21.051 INFO:teuthology.orchestra.run.smithi078.stdout: debug paxos = 20 2024-09-21T09:20:21.051 INFO:teuthology.orchestra.run.smithi078.stdout: debug auth = 20 2024-09-21T09:20:21.051 INFO:teuthology.orchestra.run.smithi078.stdout: mon data avail warn = 5 2024-09-21T09:20:21.051 INFO:teuthology.orchestra.run.smithi078.stdout: mon mgr mkfs grace = 240 2024-09-21T09:20:21.051 INFO:teuthology.orchestra.run.smithi078.stdout: mon reweight min pgs per osd = 4 2024-09-21T09:20:21.052 INFO:teuthology.orchestra.run.smithi078.stdout: mon osd reporter subtree level = osd 2024-09-21T09:20:21.052 INFO:teuthology.orchestra.run.smithi078.stdout: mon osd prime pg temp = true 2024-09-21T09:20:21.052 INFO:teuthology.orchestra.run.smithi078.stdout: mon reweight min bytes per osd = 10 2024-09-21T09:20:21.052 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:20:21.052 INFO:teuthology.orchestra.run.smithi078.stdout: # rotate auth tickets quickly to exercise renewal paths 2024-09-21T09:20:21.052 INFO:teuthology.orchestra.run.smithi078.stdout: auth mon ticket ttl = 660 # 11m 2024-09-21T09:20:21.052 INFO:teuthology.orchestra.run.smithi078.stdout: auth service ticket ttl = 240 # 4m 2024-09-21T09:20:21.052 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:20:21.052 INFO:teuthology.orchestra.run.smithi078.stdout: # don't complain about insecure global_id in the test suite 2024-09-21T09:20:21.052 INFO:teuthology.orchestra.run.smithi078.stdout: mon_warn_on_insecure_global_id_reclaim = false 2024-09-21T09:20:21.052 INFO:teuthology.orchestra.run.smithi078.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2024-09-21T09:20:21.052 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:20:21.052 INFO:teuthology.orchestra.run.smithi078.stdout: # 1m isn't quite enough 2024-09-21T09:20:21.052 INFO:teuthology.orchestra.run.smithi078.stdout: mon_down_mkfs_grace = 2m 2024-09-21T09:20:21.052 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:20:21.053 INFO:teuthology.orchestra.run.smithi078.stdout: mon_warn_on_filestore_osds = false 2024-09-21T09:20:21.053 INFO:teuthology.orchestra.run.smithi078.stdout: mon down mkfs grace = 300 2024-09-21T09:20:21.053 INFO:teuthology.orchestra.run.smithi078.stdout: mon warn on osd down out interval zero = False 2024-09-21T09:20:21.053 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:20:21.053 INFO:teuthology.orchestra.run.smithi078.stdout:[client] 2024-09-21T09:20:21.053 INFO:teuthology.orchestra.run.smithi078.stdout: rgw cache enabled = true 2024-09-21T09:20:21.053 INFO:teuthology.orchestra.run.smithi078.stdout: rgw enable ops log = true 2024-09-21T09:20:21.053 INFO:teuthology.orchestra.run.smithi078.stdout: rgw enable usage log = true 2024-09-21T09:20:21.053 INFO:teuthology.orchestra.run.smithi078.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2024-09-21T09:20:21.053 INFO:teuthology.orchestra.run.smithi078.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2024-09-21T09:20:21.053 INFO:teuthology.orchestra.run.smithi078.stdout:[mon.a] 2024-09-21T09:20:21.053 INFO:teuthology.orchestra.run.smithi078.stdout:[mon.b] 2024-09-21T09:20:21.053 INFO:teuthology.orchestra.run.smithi078.stdout:[mon.c] 2024-09-21T09:20:21.054 INFO:tasks.ceph:Creating admin key on mon.a... 2024-09-21T09:20:21.055 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' /etc/ceph/ceph.keyring 2024-09-21T09:20:21.148 INFO:tasks.ceph:Copying monmap to all nodes... 2024-09-21T09:20:21.148 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-21T09:20:21.148 DEBUG:teuthology.orchestra.run.smithi078:> dd if=/etc/ceph/ceph.keyring of=/dev/stdout 2024-09-21T09:20:21.164 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-21T09:20:21.164 DEBUG:teuthology.orchestra.run.smithi078:> dd if=/home/ubuntu/cephtest/ceph.monmap of=/dev/stdout 2024-09-21T09:20:21.219 INFO:tasks.ceph:Sending monmap to node ubuntu@smithi078.front.sepia.ceph.com 2024-09-21T09:20:21.219 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-21T09:20:21.219 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd of=/etc/ceph/ceph.keyring 2024-09-21T09:20:21.219 DEBUG:teuthology.orchestra.run.smithi078:> sudo chmod 0644 /etc/ceph/ceph.keyring 2024-09-21T09:20:21.293 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-21T09:20:21.293 DEBUG:teuthology.orchestra.run.smithi078:> dd of=/home/ubuntu/cephtest/ceph.monmap 2024-09-21T09:20:21.348 INFO:tasks.ceph:Setting up mon nodes... 2024-09-21T09:20:21.349 INFO:tasks.ceph:Setting up mgr nodes... 2024-09-21T09:20:21.349 DEBUG:teuthology.orchestra.run.smithi078:> sudo mkdir -p /var/lib/ceph/mgr/ceph-x && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mgr.x /var/lib/ceph/mgr/ceph-x/keyring 2024-09-21T09:20:21.439 INFO:teuthology.orchestra.run.smithi078.stdout:creating /var/lib/ceph/mgr/ceph-x/keyring 2024-09-21T09:20:21.442 DEBUG:teuthology.orchestra.run.smithi078:> sudo mkdir -p /var/lib/ceph/mgr/ceph-y && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mgr.y /var/lib/ceph/mgr/ceph-y/keyring 2024-09-21T09:20:21.492 INFO:teuthology.orchestra.run.smithi078.stdout:creating /var/lib/ceph/mgr/ceph-y/keyring 2024-09-21T09:20:21.494 INFO:tasks.ceph:Setting up mds nodes... 2024-09-21T09:20:21.495 INFO:tasks.ceph_client:Setting up client nodes... 2024-09-21T09:20:21.495 INFO:tasks.ceph:Running mkfs on osd nodes... 2024-09-21T09:20:21.495 INFO:tasks.ceph:ctx.disk_config.remote_to_roles_to_dev: {Remote(name='ubuntu@smithi078.front.sepia.ceph.com'): {'osd.0': '/dev/vg_nvme/lv_1', 'osd.1': '/dev/vg_nvme/lv_2', 'osd.2': '/dev/vg_nvme/lv_3', 'osd.3': '/dev/vg_nvme/lv_4'}} 2024-09-21T09:20:21.495 DEBUG:teuthology.orchestra.run.smithi078:> sudo mkdir -p /var/lib/ceph/osd/ceph-0 2024-09-21T09:20:21.519 INFO:tasks.ceph:roles_to_devs: {'osd.0': '/dev/vg_nvme/lv_1', 'osd.1': '/dev/vg_nvme/lv_2', 'osd.2': '/dev/vg_nvme/lv_3', 'osd.3': '/dev/vg_nvme/lv_4'} 2024-09-21T09:20:21.519 INFO:tasks.ceph:role: osd.0 2024-09-21T09:20:21.519 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_1 on ubuntu@smithi078.front.sepia.ceph.com 2024-09-21T09:20:21.519 DEBUG:teuthology.orchestra.run.smithi078:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_1 2024-09-21T09:20:21.591 INFO:teuthology.orchestra.run.smithi078.stdout:meta-data=/dev/vg_nvme/lv_1 isize=2048 agcount=4, agsize=5859072 blks 2024-09-21T09:20:21.591 INFO:teuthology.orchestra.run.smithi078.stdout: = sectsz=512 attr=2, projid32bit=1 2024-09-21T09:20:21.591 INFO:teuthology.orchestra.run.smithi078.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2024-09-21T09:20:21.591 INFO:teuthology.orchestra.run.smithi078.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2024-09-21T09:20:21.591 INFO:teuthology.orchestra.run.smithi078.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2024-09-21T09:20:21.591 INFO:teuthology.orchestra.run.smithi078.stdout: = sunit=0 swidth=0 blks 2024-09-21T09:20:21.591 INFO:teuthology.orchestra.run.smithi078.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2024-09-21T09:20:21.591 INFO:teuthology.orchestra.run.smithi078.stdout:log =internal log bsize=4096 blocks=16384, version=2 2024-09-21T09:20:21.592 INFO:teuthology.orchestra.run.smithi078.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2024-09-21T09:20:21.592 INFO:teuthology.orchestra.run.smithi078.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2024-09-21T09:20:22.100 INFO:teuthology.orchestra.run.smithi078.stdout:Discarding blocks...Done. 2024-09-21T09:20:22.102 INFO:tasks.ceph:mount /dev/vg_nvme/lv_1 on ubuntu@smithi078.front.sepia.ceph.com -o noatime 2024-09-21T09:20:22.102 DEBUG:teuthology.orchestra.run.smithi078:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-0 2024-09-21T09:20:22.233 DEBUG:teuthology.orchestra.run.smithi078:> sudo /sbin/restorecon /var/lib/ceph/osd/ceph-0 2024-09-21T09:20:22.265 DEBUG:teuthology.orchestra.run.smithi078:> sudo mkdir -p /var/lib/ceph/osd/ceph-1 2024-09-21T09:20:22.331 INFO:tasks.ceph:roles_to_devs: {'osd.0': '/dev/vg_nvme/lv_1', 'osd.1': '/dev/vg_nvme/lv_2', 'osd.2': '/dev/vg_nvme/lv_3', 'osd.3': '/dev/vg_nvme/lv_4'} 2024-09-21T09:20:22.331 INFO:tasks.ceph:role: osd.1 2024-09-21T09:20:22.331 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_2 on ubuntu@smithi078.front.sepia.ceph.com 2024-09-21T09:20:22.331 DEBUG:teuthology.orchestra.run.smithi078:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_2 2024-09-21T09:20:22.403 INFO:teuthology.orchestra.run.smithi078.stdout:meta-data=/dev/vg_nvme/lv_2 isize=2048 agcount=4, agsize=5859072 blks 2024-09-21T09:20:22.403 INFO:teuthology.orchestra.run.smithi078.stdout: = sectsz=512 attr=2, projid32bit=1 2024-09-21T09:20:22.403 INFO:teuthology.orchestra.run.smithi078.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2024-09-21T09:20:22.403 INFO:teuthology.orchestra.run.smithi078.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2024-09-21T09:20:22.403 INFO:teuthology.orchestra.run.smithi078.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2024-09-21T09:20:22.403 INFO:teuthology.orchestra.run.smithi078.stdout: = sunit=0 swidth=0 blks 2024-09-21T09:20:22.403 INFO:teuthology.orchestra.run.smithi078.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2024-09-21T09:20:22.403 INFO:teuthology.orchestra.run.smithi078.stdout:log =internal log bsize=4096 blocks=16384, version=2 2024-09-21T09:20:22.403 INFO:teuthology.orchestra.run.smithi078.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2024-09-21T09:20:22.404 INFO:teuthology.orchestra.run.smithi078.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2024-09-21T09:20:22.872 INFO:teuthology.orchestra.run.smithi078.stdout:Discarding blocks...Done. 2024-09-21T09:20:22.875 INFO:tasks.ceph:mount /dev/vg_nvme/lv_2 on ubuntu@smithi078.front.sepia.ceph.com -o noatime 2024-09-21T09:20:22.875 DEBUG:teuthology.orchestra.run.smithi078:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-1 2024-09-21T09:20:22.972 DEBUG:teuthology.orchestra.run.smithi078:> sudo /sbin/restorecon /var/lib/ceph/osd/ceph-1 2024-09-21T09:20:23.001 DEBUG:teuthology.orchestra.run.smithi078:> sudo mkdir -p /var/lib/ceph/osd/ceph-2 2024-09-21T09:20:23.067 INFO:tasks.ceph:roles_to_devs: {'osd.0': '/dev/vg_nvme/lv_1', 'osd.1': '/dev/vg_nvme/lv_2', 'osd.2': '/dev/vg_nvme/lv_3', 'osd.3': '/dev/vg_nvme/lv_4'} 2024-09-21T09:20:23.067 INFO:tasks.ceph:role: osd.2 2024-09-21T09:20:23.067 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_3 on ubuntu@smithi078.front.sepia.ceph.com 2024-09-21T09:20:23.067 DEBUG:teuthology.orchestra.run.smithi078:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_3 2024-09-21T09:20:23.139 INFO:teuthology.orchestra.run.smithi078.stdout:meta-data=/dev/vg_nvme/lv_3 isize=2048 agcount=4, agsize=5859072 blks 2024-09-21T09:20:23.139 INFO:teuthology.orchestra.run.smithi078.stdout: = sectsz=512 attr=2, projid32bit=1 2024-09-21T09:20:23.139 INFO:teuthology.orchestra.run.smithi078.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2024-09-21T09:20:23.139 INFO:teuthology.orchestra.run.smithi078.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2024-09-21T09:20:23.139 INFO:teuthology.orchestra.run.smithi078.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2024-09-21T09:20:23.139 INFO:teuthology.orchestra.run.smithi078.stdout: = sunit=0 swidth=0 blks 2024-09-21T09:20:23.139 INFO:teuthology.orchestra.run.smithi078.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2024-09-21T09:20:23.139 INFO:teuthology.orchestra.run.smithi078.stdout:log =internal log bsize=4096 blocks=16384, version=2 2024-09-21T09:20:23.139 INFO:teuthology.orchestra.run.smithi078.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2024-09-21T09:20:23.139 INFO:teuthology.orchestra.run.smithi078.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2024-09-21T09:20:23.628 INFO:teuthology.orchestra.run.smithi078.stdout:Discarding blocks...Done. 2024-09-21T09:20:23.630 INFO:tasks.ceph:mount /dev/vg_nvme/lv_3 on ubuntu@smithi078.front.sepia.ceph.com -o noatime 2024-09-21T09:20:23.630 DEBUG:teuthology.orchestra.run.smithi078:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-2 2024-09-21T09:20:23.769 DEBUG:teuthology.orchestra.run.smithi078:> sudo /sbin/restorecon /var/lib/ceph/osd/ceph-2 2024-09-21T09:20:23.800 DEBUG:teuthology.orchestra.run.smithi078:> sudo mkdir -p /var/lib/ceph/osd/ceph-3 2024-09-21T09:20:23.865 INFO:tasks.ceph:roles_to_devs: {'osd.0': '/dev/vg_nvme/lv_1', 'osd.1': '/dev/vg_nvme/lv_2', 'osd.2': '/dev/vg_nvme/lv_3', 'osd.3': '/dev/vg_nvme/lv_4'} 2024-09-21T09:20:23.865 INFO:tasks.ceph:role: osd.3 2024-09-21T09:20:23.865 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_4 on ubuntu@smithi078.front.sepia.ceph.com 2024-09-21T09:20:23.865 DEBUG:teuthology.orchestra.run.smithi078:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_4 2024-09-21T09:20:23.937 INFO:teuthology.orchestra.run.smithi078.stdout:meta-data=/dev/vg_nvme/lv_4 isize=2048 agcount=4, agsize=5859072 blks 2024-09-21T09:20:23.938 INFO:teuthology.orchestra.run.smithi078.stdout: = sectsz=512 attr=2, projid32bit=1 2024-09-21T09:20:23.938 INFO:teuthology.orchestra.run.smithi078.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2024-09-21T09:20:23.938 INFO:teuthology.orchestra.run.smithi078.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2024-09-21T09:20:23.938 INFO:teuthology.orchestra.run.smithi078.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2024-09-21T09:20:23.938 INFO:teuthology.orchestra.run.smithi078.stdout: = sunit=0 swidth=0 blks 2024-09-21T09:20:23.938 INFO:teuthology.orchestra.run.smithi078.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2024-09-21T09:20:23.938 INFO:teuthology.orchestra.run.smithi078.stdout:log =internal log bsize=4096 blocks=16384, version=2 2024-09-21T09:20:23.938 INFO:teuthology.orchestra.run.smithi078.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2024-09-21T09:20:23.938 INFO:teuthology.orchestra.run.smithi078.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2024-09-21T09:20:24.456 INFO:teuthology.orchestra.run.smithi078.stdout:Discarding blocks...Done. 2024-09-21T09:20:24.458 INFO:tasks.ceph:mount /dev/vg_nvme/lv_4 on ubuntu@smithi078.front.sepia.ceph.com -o noatime 2024-09-21T09:20:24.458 DEBUG:teuthology.orchestra.run.smithi078:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-3 2024-09-21T09:20:24.604 DEBUG:teuthology.orchestra.run.smithi078:> sudo /sbin/restorecon /var/lib/ceph/osd/ceph-3 2024-09-21T09:20:24.634 DEBUG:teuthology.orchestra.run.smithi078:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster ceph --mkfs --mkkey -i 0 --monmap /home/ubuntu/cephtest/ceph.monmap 2024-09-21T09:20:24.728 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:24.725+0000 7faf8d60e640 -1 auth: error reading file: /var/lib/ceph/osd/ceph-0/keyring: can't open /var/lib/ceph/osd/ceph-0/keyring: (2) No such file or directory 2024-09-21T09:20:24.728 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:24.725+0000 7faf8d60e640 -1 created new key in keyring /var/lib/ceph/osd/ceph-0/keyring 2024-09-21T09:20:24.728 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:24.725+0000 7faf8d60e640 -1 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-0/block: (2) No such file or directory 2024-09-21T09:20:24.728 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:24.725+0000 7faf8d60e640 -1 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-0/block: (2) No such file or directory 2024-09-21T09:20:24.728 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:24.725+0000 7faf8d60e640 -1 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-0/block: (2) No such file or directory 2024-09-21T09:20:24.728 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:24.726+0000 7faf8d60e640 -1 bluestore(/var/lib/ceph/osd/ceph-0) _read_fsid unparsable uuid 2024-09-21T09:20:27.580 DEBUG:teuthology.orchestra.run.smithi078:> sudo chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-21T09:20:27.607 DEBUG:teuthology.orchestra.run.smithi078:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster ceph --mkfs --mkkey -i 1 --monmap /home/ubuntu/cephtest/ceph.monmap 2024-09-21T09:20:27.697 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:27.694+0000 7fa596a1e640 -1 auth: error reading file: /var/lib/ceph/osd/ceph-1/keyring: can't open /var/lib/ceph/osd/ceph-1/keyring: (2) No such file or directory 2024-09-21T09:20:27.697 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:27.694+0000 7fa596a1e640 -1 created new key in keyring /var/lib/ceph/osd/ceph-1/keyring 2024-09-21T09:20:27.697 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:27.694+0000 7fa596a1e640 -1 bluestore(/var/lib/ceph/osd/ceph-1/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-1/block: (2) No such file or directory 2024-09-21T09:20:27.697 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:27.694+0000 7fa596a1e640 -1 bluestore(/var/lib/ceph/osd/ceph-1/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-1/block: (2) No such file or directory 2024-09-21T09:20:27.697 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:27.694+0000 7fa596a1e640 -1 bluestore(/var/lib/ceph/osd/ceph-1/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-1/block: (2) No such file or directory 2024-09-21T09:20:27.697 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:27.695+0000 7fa596a1e640 -1 bluestore(/var/lib/ceph/osd/ceph-1) _read_fsid unparsable uuid 2024-09-21T09:20:30.539 DEBUG:teuthology.orchestra.run.smithi078:> sudo chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-21T09:20:30.566 DEBUG:teuthology.orchestra.run.smithi078:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster ceph --mkfs --mkkey -i 2 --monmap /home/ubuntu/cephtest/ceph.monmap 2024-09-21T09:20:30.653 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:30.650+0000 7fa21880e640 -1 auth: error reading file: /var/lib/ceph/osd/ceph-2/keyring: can't open /var/lib/ceph/osd/ceph-2/keyring: (2) No such file or directory 2024-09-21T09:20:30.653 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:30.650+0000 7fa21880e640 -1 created new key in keyring /var/lib/ceph/osd/ceph-2/keyring 2024-09-21T09:20:30.653 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:30.650+0000 7fa21880e640 -1 bluestore(/var/lib/ceph/osd/ceph-2/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-2/block: (2) No such file or directory 2024-09-21T09:20:30.653 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:30.650+0000 7fa21880e640 -1 bluestore(/var/lib/ceph/osd/ceph-2/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-2/block: (2) No such file or directory 2024-09-21T09:20:30.653 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:30.651+0000 7fa21880e640 -1 bluestore(/var/lib/ceph/osd/ceph-2/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-2/block: (2) No such file or directory 2024-09-21T09:20:30.654 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:30.651+0000 7fa21880e640 -1 bluestore(/var/lib/ceph/osd/ceph-2) _read_fsid unparsable uuid 2024-09-21T09:20:33.500 DEBUG:teuthology.orchestra.run.smithi078:> sudo chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-21T09:20:33.528 DEBUG:teuthology.orchestra.run.smithi078:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster ceph --mkfs --mkkey -i 3 --monmap /home/ubuntu/cephtest/ceph.monmap 2024-09-21T09:20:33.616 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:33.613+0000 7f4193945640 -1 auth: error reading file: /var/lib/ceph/osd/ceph-3/keyring: can't open /var/lib/ceph/osd/ceph-3/keyring: (2) No such file or directory 2024-09-21T09:20:33.616 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:33.614+0000 7f4193945640 -1 created new key in keyring /var/lib/ceph/osd/ceph-3/keyring 2024-09-21T09:20:33.616 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:33.614+0000 7f4193945640 -1 bluestore(/var/lib/ceph/osd/ceph-3/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-3/block: (2) No such file or directory 2024-09-21T09:20:33.616 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:33.614+0000 7f4193945640 -1 bluestore(/var/lib/ceph/osd/ceph-3/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-3/block: (2) No such file or directory 2024-09-21T09:20:33.616 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:33.614+0000 7f4193945640 -1 bluestore(/var/lib/ceph/osd/ceph-3/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-3/block: (2) No such file or directory 2024-09-21T09:20:33.617 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:33.615+0000 7f4193945640 -1 bluestore(/var/lib/ceph/osd/ceph-3) _read_fsid unparsable uuid 2024-09-21T09:20:36.470 DEBUG:teuthology.orchestra.run.smithi078:> sudo chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-21T09:20:36.498 INFO:tasks.ceph:Reading keys from all nodes... 2024-09-21T09:20:36.498 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-21T09:20:36.498 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd if=/var/lib/ceph/mgr/ceph-x/keyring of=/dev/stdout 2024-09-21T09:20:36.566 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-21T09:20:36.566 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd if=/var/lib/ceph/mgr/ceph-y/keyring of=/dev/stdout 2024-09-21T09:20:36.633 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-21T09:20:36.633 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd if=/var/lib/ceph/osd/ceph-0/keyring of=/dev/stdout 2024-09-21T09:20:36.699 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-21T09:20:36.699 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd if=/var/lib/ceph/osd/ceph-1/keyring of=/dev/stdout 2024-09-21T09:20:36.765 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-21T09:20:36.765 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd if=/var/lib/ceph/osd/ceph-2/keyring of=/dev/stdout 2024-09-21T09:20:36.830 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-21T09:20:36.831 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd if=/var/lib/ceph/osd/ceph-3/keyring of=/dev/stdout 2024-09-21T09:20:36.896 INFO:tasks.ceph:Adding keys to all mons... 2024-09-21T09:20:36.896 DEBUG:teuthology.orchestra.run.smithi078:> sudo tee -a /etc/ceph/ceph.keyring 2024-09-21T09:20:36.960 INFO:teuthology.orchestra.run.smithi078.stdout:[mgr.x] 2024-09-21T09:20:36.960 INFO:teuthology.orchestra.run.smithi078.stdout: key = AQDVj+5mrKUoGhAAg6liP8sBxE5riR078DqpfQ== 2024-09-21T09:20:36.960 INFO:teuthology.orchestra.run.smithi078.stdout:[mgr.y] 2024-09-21T09:20:36.960 INFO:teuthology.orchestra.run.smithi078.stdout: key = AQDVj+5m1+1MHRAAYFUYUmhGTxlhdYeC7j2rig== 2024-09-21T09:20:36.960 INFO:teuthology.orchestra.run.smithi078.stdout:[osd.0] 2024-09-21T09:20:36.960 INFO:teuthology.orchestra.run.smithi078.stdout: key = AQDYj+5m75pTKxAAAL/nVym1+55sc/v2pEuQmw== 2024-09-21T09:20:36.960 INFO:teuthology.orchestra.run.smithi078.stdout:[osd.1] 2024-09-21T09:20:36.961 INFO:teuthology.orchestra.run.smithi078.stdout: key = AQDbj+5mOah4KRAAYpdP13Z8bNkAuxvvykJRnw== 2024-09-21T09:20:36.961 INFO:teuthology.orchestra.run.smithi078.stdout:[osd.2] 2024-09-21T09:20:36.961 INFO:teuthology.orchestra.run.smithi078.stdout: key = AQDej+5m0JfdJhAAfywNHNX634p2+JJuIWOzCg== 2024-09-21T09:20:36.961 INFO:teuthology.orchestra.run.smithi078.stdout:[osd.3] 2024-09-21T09:20:36.961 INFO:teuthology.orchestra.run.smithi078.stdout: key = AQDhj+5mfzGtJBAA/EPfL5wDJgEO/oCCCt5hoA== 2024-09-21T09:20:36.962 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=mgr.x --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2024-09-21T09:20:37.053 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=mgr.y --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2024-09-21T09:20:37.103 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.0 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-09-21T09:20:37.153 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.1 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-09-21T09:20:37.203 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.2 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-09-21T09:20:37.252 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.3 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-09-21T09:20:37.302 INFO:tasks.ceph:Running mkfs on mon nodes... 2024-09-21T09:20:37.302 DEBUG:teuthology.orchestra.run.smithi078:> sudo mkdir -p /var/lib/ceph/mon/ceph-a 2024-09-21T09:20:37.326 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster ceph --mkfs -i a --monmap /home/ubuntu/cephtest/ceph.monmap --keyring /etc/ceph/ceph.keyring 2024-09-21T09:20:37.421 DEBUG:teuthology.orchestra.run.smithi078:> sudo chown -R ceph:ceph /var/lib/ceph/mon/ceph-a 2024-09-21T09:20:37.445 DEBUG:teuthology.orchestra.run.smithi078:> sudo mkdir -p /var/lib/ceph/mon/ceph-b 2024-09-21T09:20:37.509 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster ceph --mkfs -i b --monmap /home/ubuntu/cephtest/ceph.monmap --keyring /etc/ceph/ceph.keyring 2024-09-21T09:20:37.604 DEBUG:teuthology.orchestra.run.smithi078:> sudo chown -R ceph:ceph /var/lib/ceph/mon/ceph-b 2024-09-21T09:20:37.629 DEBUG:teuthology.orchestra.run.smithi078:> sudo mkdir -p /var/lib/ceph/mon/ceph-c 2024-09-21T09:20:37.693 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster ceph --mkfs -i c --monmap /home/ubuntu/cephtest/ceph.monmap --keyring /etc/ceph/ceph.keyring 2024-09-21T09:20:37.788 DEBUG:teuthology.orchestra.run.smithi078:> sudo chown -R ceph:ceph /var/lib/ceph/mon/ceph-c 2024-09-21T09:20:37.812 DEBUG:teuthology.orchestra.run.smithi078:> rm -- /home/ubuntu/cephtest/ceph.monmap 2024-09-21T09:20:37.867 INFO:tasks.ceph:Starting mon daemons in cluster ceph... 2024-09-21T09:20:37.867 INFO:tasks.ceph.mon.a:Restarting daemon 2024-09-21T09:20:37.868 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster ceph -i a 2024-09-21T09:20:37.909 INFO:tasks.ceph.mon.a:Started 2024-09-21T09:20:37.910 INFO:tasks.ceph.mon.b:Restarting daemon 2024-09-21T09:20:37.910 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster ceph -i b 2024-09-21T09:20:37.912 INFO:tasks.ceph.mon.b:Started 2024-09-21T09:20:37.912 INFO:tasks.ceph.mon.c:Restarting daemon 2024-09-21T09:20:37.912 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster ceph -i c 2024-09-21T09:20:37.914 INFO:tasks.ceph.mon.c:Started 2024-09-21T09:20:37.914 INFO:tasks.ceph:Starting mgr daemons in cluster ceph... 2024-09-21T09:20:37.914 INFO:tasks.ceph.mgr.x:Restarting daemon 2024-09-21T09:20:37.914 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster ceph -i x 2024-09-21T09:20:37.916 INFO:tasks.ceph.mgr.x:Started 2024-09-21T09:20:37.916 INFO:tasks.ceph.mgr.y:Restarting daemon 2024-09-21T09:20:37.917 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster ceph -i y 2024-09-21T09:20:37.919 INFO:tasks.ceph.mgr.y:Started 2024-09-21T09:20:37.919 INFO:tasks.ceph:setting config [client] client_mount_timeout = 600 2024-09-21T09:20:37.919 DEBUG:teuthology.orchestra.run.smithi078:> sudo ceph --cluster ceph config set client client_mount_timeout 600 2024-09-21T09:20:37.921 INFO:tasks.ceph:setting config [client] debug_client = 20 2024-09-21T09:20:37.921 DEBUG:teuthology.orchestra.run.smithi078:> sudo ceph --cluster ceph config set client debug_client 20 2024-09-21T09:20:37.965 INFO:tasks.ceph:setting config [client] debug_ms = 1 2024-09-21T09:20:37.965 DEBUG:teuthology.orchestra.run.smithi078:> sudo ceph --cluster ceph config set client debug_ms 1 2024-09-21T09:20:38.008 INFO:tasks.ceph:setting config [client] rados_mon_op_timeout = 900 2024-09-21T09:20:38.008 DEBUG:teuthology.orchestra.run.smithi078:> sudo ceph --cluster ceph config set client rados_mon_op_timeout 900 2024-09-21T09:20:38.052 INFO:tasks.ceph:setting config [client] rados_osd_op_timeout = 900 2024-09-21T09:20:38.052 DEBUG:teuthology.orchestra.run.smithi078:> sudo ceph --cluster ceph config set client rados_osd_op_timeout 900 2024-09-21T09:20:38.095 INFO:tasks.ceph:setting config [mds] debug_mds = 20 2024-09-21T09:20:38.095 DEBUG:teuthology.orchestra.run.smithi078:> sudo ceph --cluster ceph config set mds debug_mds 20 2024-09-21T09:20:38.137 INFO:tasks.ceph:setting config [mds] debug_mds_balancer = 20 2024-09-21T09:20:38.138 DEBUG:teuthology.orchestra.run.smithi078:> sudo ceph --cluster ceph config set mds debug_mds_balancer 20 2024-09-21T09:20:38.180 INFO:tasks.ceph:setting config [mds] debug_ms = 2 2024-09-21T09:20:38.180 DEBUG:teuthology.orchestra.run.smithi078:> sudo ceph --cluster ceph config set mds debug_ms 2 2024-09-21T09:20:38.227 INFO:tasks.ceph:setting config [mds] mds_debug_frag = True 2024-09-21T09:20:38.227 DEBUG:teuthology.orchestra.run.smithi078:> sudo ceph --cluster ceph config set mds mds_debug_frag True 2024-09-21T09:20:38.271 INFO:tasks.ceph:setting config [mds] mds_debug_scatterstat = True 2024-09-21T09:20:38.271 DEBUG:teuthology.orchestra.run.smithi078:> sudo ceph --cluster ceph config set mds mds_debug_scatterstat True 2024-09-21T09:20:38.315 INFO:tasks.ceph:setting config [mds] mds_op_complaint_time = 180 2024-09-21T09:20:38.315 DEBUG:teuthology.orchestra.run.smithi078:> sudo ceph --cluster ceph config set mds mds_op_complaint_time 180 2024-09-21T09:20:38.358 INFO:tasks.ceph:setting config [mds] mds_verify_scatter = True 2024-09-21T09:20:38.358 DEBUG:teuthology.orchestra.run.smithi078:> sudo ceph --cluster ceph config set mds mds_verify_scatter True 2024-09-21T09:20:38.401 INFO:tasks.ceph:setting config [mds] osd_op_complaint_time = 180 2024-09-21T09:20:38.401 DEBUG:teuthology.orchestra.run.smithi078:> sudo ceph --cluster ceph config set mds osd_op_complaint_time 180 2024-09-21T09:20:38.444 INFO:tasks.ceph:setting config [mds] rados_mon_op_timeout = 900 2024-09-21T09:20:38.444 DEBUG:teuthology.orchestra.run.smithi078:> sudo ceph --cluster ceph config set mds rados_mon_op_timeout 900 2024-09-21T09:20:38.448 INFO:tasks.ceph:setting config [mds] rados_osd_op_timeout = 900 2024-09-21T09:20:38.448 DEBUG:teuthology.orchestra.run.smithi078:> sudo ceph --cluster ceph config set mds rados_osd_op_timeout 900 2024-09-21T09:20:38.487 INFO:tasks.ceph:setting config [mon] mon_op_complaint_time = 120 2024-09-21T09:20:38.488 DEBUG:teuthology.orchestra.run.smithi078:> sudo ceph --cluster ceph config set mon mon_op_complaint_time 120 2024-09-21T09:20:38.488 INFO:tasks.ceph.mgr.x.smithi078.stderr:2024-09-21T09:20:38.484+0000 7fda56037040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-21T09:20:38.497 INFO:tasks.ceph.mgr.y.smithi078.stderr:2024-09-21T09:20:38.495+0000 7ff872721040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-21T09:20:38.500 INFO:tasks.ceph:setting config [mon] mon_warn_on_pool_no_app = False 2024-09-21T09:20:38.500 DEBUG:teuthology.orchestra.run.smithi078:> sudo ceph --cluster ceph config set mon mon_warn_on_pool_no_app False 2024-09-21T09:20:38.509 INFO:tasks.ceph:setting config [osd] osd_op_complaint_time = 180 2024-09-21T09:20:38.509 DEBUG:teuthology.orchestra.run.smithi078:> sudo ceph --cluster ceph config set osd osd_op_complaint_time 180 2024-09-21T09:20:38.529 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.527+0000 7fc578ff9640 1 -- 172.21.15.78:0/3874011602 <== mon.1 v2:172.21.15.78:3301/0 11 ==== mon_command_ack([{prefix=config set, name=debug_ms}]=0 v6)=0 v6) v1 ==== 109+0+0 (secure 0 0 0) 0x7fc56c005180 con 0x7fc57c070a80 2024-09-21T09:20:38.533 DEBUG:tasks.ceph:set 18 configs 2024-09-21T09:20:38.533 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'client', 'client_mount_timeout', '600'], check_status=True, hostname='smithi078') 2024-09-21T09:20:38.534 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.531+0000 7fc5627fc640 1 -- 172.21.15.78:0/3874011602 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fc57c070a80 msgr2=0x7fc57c117b30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:38.534 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.531+0000 7fc5627fc640 1 --2- 172.21.15.78:0/3874011602 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fc57c070a80 0x7fc57c117b30 secure :-1 s=READY pgs=9 cs=0 l=1 rev1=1 crypto rx=0x7fc56c010aa0 tx=0x7fc56c010f60 comp rx=0 tx=0).stop 2024-09-21T09:20:38.534 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.531+0000 7fc5627fc640 1 -- 172.21.15.78:0/3874011602 shutdown_connections 2024-09-21T09:20:38.534 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.531+0000 7fc5627fc640 1 --2- 172.21.15.78:0/3874011602 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fc57c1c4c20 0x7fc57c1c7010 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.534 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.531+0000 7fc5627fc640 1 --2- 172.21.15.78:0/3874011602 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fc57c118070 0x7fc57c118490 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.535 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.531+0000 7fc5627fc640 1 --2- 172.21.15.78:0/3874011602 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fc57c070a80 0x7fc57c117b30 unknown :-1 s=CLOSED pgs=9 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.535 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.531+0000 7fc5627fc640 1 -- 172.21.15.78:0/3874011602 >> 172.21.15.78:0/3874011602 conn(0x7fc57c097e40 msgr2=0x7fc57c09bb40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:20:38.535 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.531+0000 7fc5627fc640 1 -- 172.21.15.78:0/3874011602 shutdown_connections 2024-09-21T09:20:38.535 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.532+0000 7fc5627fc640 1 -- 172.21.15.78:0/3874011602 wait complete. 2024-09-21T09:20:38.569 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.567+0000 7f01b6e2b640 1 -- 172.21.15.78:0/2762091868 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f01a80b4e80 msgr2=0x7f01a80b7270 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:38.569 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.567+0000 7f01b6e2b640 1 --2- 172.21.15.78:0/2762091868 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f01a80b4e80 0x7f01a80b7270 secure :-1 s=READY pgs=18 cs=0 l=1 rev1=1 crypto rx=0x7f01a4005fd0 tx=0x7f01a402a5a0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.575 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.569+0000 7f01b6e2b640 1 -- 172.21.15.78:0/2762091868 shutdown_connections 2024-09-21T09:20:38.575 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.569+0000 7f01b6e2b640 1 --2- 172.21.15.78:0/2762091868 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f01a80b4e80 0x7f01a80b7270 unknown :-1 s=CLOSED pgs=18 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.575 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.569+0000 7f01b6e2b640 1 --2- 172.21.15.78:0/2762091868 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f01a8019370 0x7f01a80b4940 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.575 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.569+0000 7f01b6e2b640 1 -- 172.21.15.78:0/2762091868 >> 172.21.15.78:0/2762091868 conn(0x7f01a8005bf0 msgr2=0x7f01a8006000 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:20:38.575 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.572+0000 7f01b6e2b640 1 -- 172.21.15.78:0/2762091868 shutdown_connections 2024-09-21T09:20:38.575 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.572+0000 7f01b6e2b640 1 -- 172.21.15.78:0/2762091868 wait complete. 2024-09-21T09:20:38.578 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.574+0000 7f01b6e2b640 1 Processor -- start 2024-09-21T09:20:38.578 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.574+0000 7f01b6e2b640 1 -- start start 2024-09-21T09:20:38.578 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.574+0000 7f01b6e2b640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f01a8018880 0x7f01a814dff0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:38.578 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.574+0000 7f01b6e2b640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f01a8019370 0x7f01a8144770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:38.578 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.574+0000 7f01b6e2b640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f01a80b4e80 0x7f01a8144cb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:38.579 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.574+0000 7f01b6e2b640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f01a8013870 con 0x7f01a80b4e80 2024-09-21T09:20:38.579 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.574+0000 7f01b6e2b640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f01a800dfe0 con 0x7f01a8019370 2024-09-21T09:20:38.579 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.574+0000 7f01b6e2b640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 v1 -- 0x7f01a800e3b0 con 0x7f01a8018880 2024-09-21T09:20:38.579 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.575+0000 7f01b4bc7640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f01a8018880 0x7f01a814dff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:38.579 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.575+0000 7f01b53c8640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f01a80b4e80 0x7f01a8144cb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:38.579 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.575+0000 7f01b4bc7640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f01a8018880 0x7f01a814dff0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3302/0 says I am v2:172.21.15.78:43086/0 (socket says 172.21.15.78:43086) 2024-09-21T09:20:38.579 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.575+0000 7f01b4bc7640 1 -- 172.21.15.78:0/1257577644 learned_addr learned my addr 172.21.15.78:0/1257577644 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:20:38.579 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.575+0000 7f01b53c8640 1 -- 172.21.15.78:0/1257577644 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f01a8018880 msgr2=0x7f01a814dff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:38.579 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.575+0000 7f01b53c8640 1 --2- 172.21.15.78:0/1257577644 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f01a8018880 0x7f01a814dff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.579 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.575+0000 7f01b53c8640 1 -- 172.21.15.78:0/1257577644 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f01a8019370 msgr2=0x7f01a8144770 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-21T09:20:38.579 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.575+0000 7f01b53c8640 1 --2- 172.21.15.78:0/1257577644 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f01a8019370 0x7f01a8144770 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.579 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.576+0000 7f01b53c8640 1 -- 172.21.15.78:0/1257577644 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f01a4009560 con 0x7f01a80b4e80 2024-09-21T09:20:38.580 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.576+0000 7f01b53c8640 1 --2- 172.21.15.78:0/1257577644 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f01a80b4e80 0x7f01a8144cb0 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7f01a8007d90 tx=0x7f01a4004a50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:20:38.583 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.576+0000 7f01ad7fa640 1 -- 172.21.15.78:0/1257577644 <== mon.0 v2:172.21.15.78:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f01a4005220 con 0x7f01a80b4e80 2024-09-21T09:20:38.584 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.576+0000 7f01ad7fa640 1 -- 172.21.15.78:0/1257577644 <== mon.0 v2:172.21.15.78:3300/0 2 ==== config(4 keys) v1 ==== 110+0+0 (secure 0 0 0) 0x7f01a4004e50 con 0x7f01a80b4e80 2024-09-21T09:20:38.584 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.576+0000 7f01ad7fa640 1 -- 172.21.15.78:0/1257577644 <== mon.0 v2:172.21.15.78:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f01a4003850 con 0x7f01a80b4e80 2024-09-21T09:20:38.584 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.576+0000 7f01b6e2b640 1 -- 172.21.15.78:0/1257577644 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f01a800d800 con 0x7f01a80b4e80 2024-09-21T09:20:38.584 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.576+0000 7f01b6e2b640 1 -- 172.21.15.78:0/1257577644 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f01a81451f0 con 0x7f01a80b4e80 2024-09-21T09:20:38.584 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.578+0000 7f01ad7fa640 1 -- 172.21.15.78:0/1257577644 <== mon.0 v2:172.21.15.78:3300/0 4 ==== mgrmap(e 1) v1 ==== 668+0+0 (secure 0 0 0) 0x7f01a4003cb0 con 0x7f01a80b4e80 2024-09-21T09:20:38.584 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.578+0000 7f01ad7fa640 1 -- 172.21.15.78:0/1257577644 <== mon.0 v2:172.21.15.78:3300/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 725+0+0 (secure 0 0 0) 0x7f01a4045790 con 0x7f01a80b4e80 2024-09-21T09:20:38.584 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.581+0000 7f01b6e2b640 1 -- 172.21.15.78:0/1257577644 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0190005250 con 0x7f01a80b4e80 2024-09-21T09:20:38.598 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.585+0000 7f01ad7fa640 1 -- 172.21.15.78:0/1257577644 <== mon.0 v2:172.21.15.78:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+73245 (secure 0 0 0) 0x7f01a404a060 con 0x7f01a80b4e80 2024-09-21T09:20:38.605 INFO:tasks.ceph.mgr.y.smithi078.stderr:2024-09-21T09:20:38.603+0000 7ff872721040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-21T09:20:38.606 INFO:tasks.ceph.mgr.x.smithi078.stderr:2024-09-21T09:20:38.604+0000 7fda56037040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-21T09:20:38.659 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.656+0000 7f143445a640 1 -- 172.21.15.78:0/155652908 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f142c12ced0 msgr2=0x7f142c06c520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:38.659 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.656+0000 7f143445a640 1 --2- 172.21.15.78:0/155652908 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f142c12ced0 0x7f142c06c520 secure :-1 s=READY pgs=15 cs=0 l=1 rev1=1 crypto rx=0x7f142800dae0 tx=0x7f142802b130 comp rx=0 tx=0).stop 2024-09-21T09:20:38.659 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.657+0000 7f143445a640 1 -- 172.21.15.78:0/155652908 shutdown_connections 2024-09-21T09:20:38.659 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.657+0000 7f143445a640 1 --2- 172.21.15.78:0/155652908 >> v2:172.21.15.78:3300/0 conn(0x7f142c06ca60 0x7f142c06cef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.659 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.657+0000 7f143445a640 1 --2- 172.21.15.78:0/155652908 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f142c12ced0 0x7f142c06c520 unknown :-1 s=CLOSED pgs=15 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.660 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.657+0000 7f143445a640 1 --2- 172.21.15.78:0/155652908 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f142c12c5b0 0x7f142c12c990 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.660 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.657+0000 7f143445a640 1 -- 172.21.15.78:0/155652908 >> 172.21.15.78:0/155652908 conn(0x7f142c09e9d0 msgr2=0x7f142c09ede0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:20:38.661 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.657+0000 7f143445a640 1 -- 172.21.15.78:0/155652908 shutdown_connections 2024-09-21T09:20:38.661 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.657+0000 7f143445a640 1 -- 172.21.15.78:0/155652908 wait complete. 2024-09-21T09:20:38.661 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.659+0000 7f143445a640 1 Processor -- start 2024-09-21T09:20:38.661 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.659+0000 7f143445a640 1 -- start start 2024-09-21T09:20:38.663 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.660+0000 7f143445a640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f142c06ca60 0x7f142c12d2b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:38.663 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.660+0000 7f143445a640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f142c12c5b0 0x7f142c12d7f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:38.663 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.660+0000 7f14321f6640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f142c06ca60 0x7f142c12d2b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:38.663 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.660+0000 7f143445a640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f142c12ced0 0x7f142c12dd30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:38.663 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.660+0000 7f14321f6640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f142c06ca60 0x7f142c12d2b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3301/0 says I am v2:172.21.15.78:50146/0 (socket says 172.21.15.78:50146) 2024-09-21T09:20:38.663 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.660+0000 7f143445a640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f142c0718f0 con 0x7f142c12ced0 2024-09-21T09:20:38.663 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.660+0000 7f143445a640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f142c126ac0 con 0x7f142c06ca60 2024-09-21T09:20:38.663 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.660+0000 7f143445a640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 v1 -- 0x7f142c070fc0 con 0x7f142c12c5b0 2024-09-21T09:20:38.663 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.660+0000 7f14319f5640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f142c12c5b0 0x7f142c12d7f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:38.663 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.660+0000 7f14319f5640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f142c12c5b0 0x7f142c12d7f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3302/0 says I am v2:172.21.15.78:43108/0 (socket says 172.21.15.78:43108) 2024-09-21T09:20:38.663 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.660+0000 7f14319f5640 1 -- 172.21.15.78:0/2700596629 learned_addr learned my addr 172.21.15.78:0/2700596629 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:20:38.664 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.661+0000 7f14321f6640 1 -- 172.21.15.78:0/2700596629 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f142c12c5b0 msgr2=0x7f142c12d7f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:38.664 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.661+0000 7f14321f6640 1 --2- 172.21.15.78:0/2700596629 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f142c12c5b0 0x7f142c12d7f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.664 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.661+0000 7f14321f6640 1 -- 172.21.15.78:0/2700596629 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f142c12ced0 msgr2=0x7f142c12dd30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:38.664 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.661+0000 7f14321f6640 1 --2- 172.21.15.78:0/2700596629 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f142c12ced0 0x7f142c12dd30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.664 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.661+0000 7f14321f6640 1 -- 172.21.15.78:0/2700596629 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f142800d740 con 0x7f142c06ca60 2024-09-21T09:20:38.664 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.662+0000 7f14321f6640 1 --2- 172.21.15.78:0/2700596629 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f142c06ca60 0x7f142c12d2b0 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7f1420004630 tx=0x7f142000da30 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:20:38.665 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.662+0000 7f141effd640 1 -- 172.21.15.78:0/2700596629 <== mon.1 v2:172.21.15.78:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1420012ae0 con 0x7f142c06ca60 2024-09-21T09:20:38.665 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.662+0000 7f141effd640 1 -- 172.21.15.78:0/2700596629 <== mon.1 v2:172.21.15.78:3301/0 2 ==== config(4 keys) v1 ==== 110+0+0 (secure 0 0 0) 0x7f1420012d10 con 0x7f142c06ca60 2024-09-21T09:20:38.665 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.663+0000 7f143445a640 1 -- 172.21.15.78:0/2700596629 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f142c071f30 con 0x7f142c06ca60 2024-09-21T09:20:38.665 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.663+0000 7f143445a640 1 -- 172.21.15.78:0/2700596629 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f142c12ecb0 con 0x7f142c06ca60 2024-09-21T09:20:38.665 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.663+0000 7f141effd640 1 -- 172.21.15.78:0/2700596629 <== mon.1 v2:172.21.15.78:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1420005280 con 0x7f142c06ca60 2024-09-21T09:20:38.667 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.665+0000 7f141effd640 1 -- 172.21.15.78:0/2700596629 <== mon.1 v2:172.21.15.78:3301/0 4 ==== mgrmap(e 1) v1 ==== 668+0+0 (secure 0 0 0) 0x7f14200047a0 con 0x7f142c06ca60 2024-09-21T09:20:38.669 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.666+0000 7f4665e53640 1 -- 172.21.15.78:0/3063733162 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f466006f420 msgr2=0x7f466006f8b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:38.669 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.666+0000 7f4665e53640 1 --2- 172.21.15.78:0/3063733162 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f466006f420 0x7f466006f8b0 secure :-1 s=READY pgs=20 cs=0 l=1 rev1=1 crypto rx=0x7f4658005fd0 tx=0x7f465802a930 comp rx=0 tx=0).stop 2024-09-21T09:20:38.670 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.665+0000 7f143445a640 1 -- 172.21.15.78:0/2700596629 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1414005250 con 0x7f142c06ca60 2024-09-21T09:20:38.671 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.668+0000 7f141effd640 1 -- 172.21.15.78:0/2700596629 <== mon.1 v2:172.21.15.78:3301/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 725+0+0 (secure 0 0 0) 0x7f1420023aa0 con 0x7f142c06ca60 2024-09-21T09:20:38.671 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.668+0000 7f141effd640 1 -- 172.21.15.78:0/2700596629 <== mon.1 v2:172.21.15.78:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+73245 (secure 0 0 0) 0x7f1420005720 con 0x7f142c06ca60 2024-09-21T09:20:38.673 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.668+0000 7f4665e53640 1 -- 172.21.15.78:0/3063733162 shutdown_connections 2024-09-21T09:20:38.673 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.668+0000 7f4665e53640 1 --2- 172.21.15.78:0/3063733162 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f4660070a30 0x7f466006eee0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.673 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.668+0000 7f4665e53640 1 --2- 172.21.15.78:0/3063733162 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f466006f420 0x7f466006f8b0 unknown :-1 s=CLOSED pgs=20 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.673 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.668+0000 7f4665e53640 1 --2- 172.21.15.78:0/3063733162 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f466012a500 0x7f466012a8e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.673 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.668+0000 7f4665e53640 1 -- 172.21.15.78:0/3063733162 >> 172.21.15.78:0/3063733162 conn(0x7f4660097190 msgr2=0x7f46600975a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:20:38.690 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.688+0000 7f1aa5ffb640 1 -- 172.21.15.78:0/3145895023 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_command([{prefix=config set, name=debug_mds_balancer}] v 0) v1 -- 0x7f1a980052b0 con 0x7f1ac006bbc0 2024-09-21T09:20:38.691 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.686+0000 7f2efc718640 1 -- 172.21.15.78:0/688862017 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command([{prefix=config set, name=debug_ms}] v 0) v1 -- 0x7f2ed00052b0 con 0x7f2ef406eb00 2024-09-21T09:20:38.692 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.686+0000 7f4665e53640 1 -- 172.21.15.78:0/3063733162 shutdown_connections 2024-09-21T09:20:38.692 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.686+0000 7f4665e53640 1 -- 172.21.15.78:0/3063733162 wait complete. 2024-09-21T09:20:38.692 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.687+0000 7f4665e53640 1 Processor -- start 2024-09-21T09:20:38.692 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.688+0000 7f4665e53640 1 -- start start 2024-09-21T09:20:38.692 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.689+0000 7f4665e53640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f466006f420 0x7f46600959e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:38.692 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.689+0000 7f4665e53640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f4660070a30 0x7f4660095f20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:38.692 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.689+0000 7f4665e53640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f466012a500 0x7f466012de10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:38.692 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.689+0000 7f4665e53640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4660130240 con 0x7f4660070a30 2024-09-21T09:20:38.692 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.689+0000 7f4665e53640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4660124810 con 0x7f466012a500 2024-09-21T09:20:38.692 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.689+0000 7f4665e53640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 v1 -- 0x7f46600893f0 con 0x7f466006f420 2024-09-21T09:20:38.693 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.690+0000 7f465effd640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f4660070a30 0x7f4660095f20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:38.693 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.690+0000 7f465effd640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f4660070a30 0x7f4660095f20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3300/0 says I am v2:172.21.15.78:34944/0 (socket says 172.21.15.78:34944) 2024-09-21T09:20:38.693 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.690+0000 7f465effd640 1 -- 172.21.15.78:0/3561683452 learned_addr learned my addr 172.21.15.78:0/3561683452 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:20:38.693 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.690+0000 7f465ffff640 1 --2- 172.21.15.78:0/3561683452 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f466012a500 0x7f466012de10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:38.694 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.690+0000 7f465ffff640 1 -- 172.21.15.78:0/3561683452 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f466006f420 msgr2=0x7f46600959e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:38.694 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.691+0000 7f465ffff640 1 --2- 172.21.15.78:0/3561683452 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f466006f420 0x7f46600959e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.694 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.691+0000 7f465ffff640 1 -- 172.21.15.78:0/3561683452 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f4660070a30 msgr2=0x7f4660095f20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:38.694 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.691+0000 7f465ffff640 1 --2- 172.21.15.78:0/3561683452 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f4660070a30 0x7f4660095f20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.694 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.691+0000 7f465ffff640 1 -- 172.21.15.78:0/3561683452 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4658009900 con 0x7f466012a500 2024-09-21T09:20:38.694 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.691+0000 7f465ffff640 1 --2- 172.21.15.78:0/3561683452 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f466012a500 0x7f466012de10 secure :-1 s=READY pgs=17 cs=0 l=1 rev1=1 crypto rx=0x7f466006ace0 tx=0x7f4658003f90 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:20:38.695 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.693+0000 7f2eeaffd640 1 -- 172.21.15.78:0/688862017 <== mon.0 v2:172.21.15.78:3300/0 10 ==== mon_command_ack([{prefix=config set, name=debug_ms}]=0 v7)=0 v7) v1 ==== 106+0+0 (secure 0 0 0) 0x7f2ef0029a60 con 0x7f2ef406eb00 2024-09-21T09:20:38.700 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.698+0000 7f82d5b76640 1 -- 172.21.15.78:0/1258504649 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_command([{prefix=config set, name=mds_debug_frag}] v 0) v1 -- 0x7f82b0001680 con 0x7f82d0085cb0 2024-09-21T09:20:38.702 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.700+0000 7f2efc718640 1 -- 172.21.15.78:0/688862017 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f2ef406eb00 msgr2=0x7f2ef41338c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:38.702 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.700+0000 7f2efc718640 1 --2- 172.21.15.78:0/688862017 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f2ef406eb00 0x7f2ef41338c0 secure :-1 s=READY pgs=17 cs=0 l=1 rev1=1 crypto rx=0x7f2ef001fd70 tx=0x7f2ef001d6b0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.704 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.700+0000 7f2efc718640 1 -- 172.21.15.78:0/688862017 shutdown_connections 2024-09-21T09:20:38.704 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.700+0000 7f2efc718640 1 --2- 172.21.15.78:0/688862017 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f2ef412c650 0x7f2ef40894f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.704 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.700+0000 7f2efc718640 1 --2- 172.21.15.78:0/688862017 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f2ef406ff10 0x7f2ef4088fb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.704 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.700+0000 7f2efc718640 1 --2- 172.21.15.78:0/688862017 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f2ef406eb00 0x7f2ef41338c0 unknown :-1 s=CLOSED pgs=17 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.704 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.700+0000 7f2efc718640 1 -- 172.21.15.78:0/688862017 >> 172.21.15.78:0/688862017 conn(0x7f2ef409c980 msgr2=0x7f2ef406d3b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:20:38.705 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.700+0000 7f2efc718640 1 -- 172.21.15.78:0/688862017 shutdown_connections 2024-09-21T09:20:38.705 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.700+0000 7f2efc718640 1 -- 172.21.15.78:0/688862017 wait complete. 2024-09-21T09:20:38.705 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.700+0000 7f4647fff640 1 -- 172.21.15.78:0/3561683452 <== mon.1 v2:172.21.15.78:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4658030830 con 0x7f466012a500 2024-09-21T09:20:38.705 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.700+0000 7f4665e53640 1 -- 172.21.15.78:0/3561683452 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f466012f840 con 0x7f466012a500 2024-09-21T09:20:38.705 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.700+0000 7f4665e53640 1 -- 172.21.15.78:0/3561683452 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4660130cb0 con 0x7f466012a500 2024-09-21T09:20:38.705 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.702+0000 7f4647fff640 1 -- 172.21.15.78:0/3561683452 <== mon.1 v2:172.21.15.78:3301/0 2 ==== config(4 keys) v1 ==== 110+0+0 (secure 0 0 0) 0x7f46580040c0 con 0x7f466012a500 2024-09-21T09:20:38.705 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.702+0000 7f4e6acc5640 1 -- 172.21.15.78:0/4138284522 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f4e64088ac0 msgr2=0x7f4e6408aed0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:38.706 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.702+0000 7f4e6acc5640 1 --2- 172.21.15.78:0/4138284522 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f4e64088ac0 0x7f4e6408aed0 secure :-1 s=READY pgs=18 cs=0 l=1 rev1=1 crypto rx=0x7f4e5c009f00 tx=0x7f4e5c02abe0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.706 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.702+0000 7f4647fff640 1 -- 172.21.15.78:0/3561683452 <== mon.1 v2:172.21.15.78:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4658004430 con 0x7f466012a500 2024-09-21T09:20:38.706 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.703+0000 7f4647fff640 1 -- 172.21.15.78:0/3561683452 <== mon.1 v2:172.21.15.78:3301/0 4 ==== mgrmap(e 1) v1 ==== 668+0+0 (secure 0 0 0) 0x7f465800a040 con 0x7f466012a500 2024-09-21T09:20:38.707 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.705+0000 7f4647fff640 1 -- 172.21.15.78:0/3561683452 <== mon.1 v2:172.21.15.78:3301/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 725+0+0 (secure 0 0 0) 0x7f4658044440 con 0x7f466012a500 2024-09-21T09:20:38.708 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.706+0000 7f4e6acc5640 1 -- 172.21.15.78:0/4138284522 shutdown_connections 2024-09-21T09:20:38.708 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.706+0000 7f4e6acc5640 1 --2- 172.21.15.78:0/4138284522 >> v2:172.21.15.78:3300/0 conn(0x7f4e6408b410 0x7f4e6408d890 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.708 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.706+0000 7f4e6acc5640 1 --2- 172.21.15.78:0/4138284522 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f4e64088ac0 0x7f4e6408aed0 unknown :-1 s=CLOSED pgs=18 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.708 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.706+0000 7f4e6acc5640 1 -- 172.21.15.78:0/4138284522 >> 172.21.15.78:0/4138284522 conn(0x7f4e640986a0 msgr2=0x7f4e64098ab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:20:38.709 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.707+0000 7f4645ffb640 1 -- 172.21.15.78:0/3561683452 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4640005250 con 0x7f466012a500 2024-09-21T09:20:38.717 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.706+0000 7f4e6acc5640 1 -- 172.21.15.78:0/4138284522 shutdown_connections 2024-09-21T09:20:38.717 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.706+0000 7f4e6acc5640 1 -- 172.21.15.78:0/4138284522 wait complete. 2024-09-21T09:20:38.717 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.707+0000 7f4e6acc5640 1 Processor -- start 2024-09-21T09:20:38.718 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.707+0000 7f4e6acc5640 1 -- start start 2024-09-21T09:20:38.718 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.712+0000 7f4e6acc5640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f4e64088ac0 0x7f4e6408e310 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:38.718 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.712+0000 7f4e6acc5640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f4e6408b410 0x7f4e6408e850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:38.718 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.712+0000 7f4e6acc5640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f4e64091f10 0x7f4e6408ed90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:38.718 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.712+0000 7f4e6acc5640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4e6408f3b0 con 0x7f4e64088ac0 2024-09-21T09:20:38.718 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.712+0000 7f4e6acc5640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4e64090460 con 0x7f4e64091f10 2024-09-21T09:20:38.718 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.712+0000 7f4e6acc5640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 v1 -- 0x7f4e640909c0 con 0x7f4e6408b410 2024-09-21T09:20:38.718 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.713+0000 7f4e63fff640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f4e6408b410 0x7f4e6408e850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:38.718 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.713+0000 7f4e63fff640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f4e6408b410 0x7f4e6408e850 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3302/0 says I am v2:172.21.15.78:43128/0 (socket says 172.21.15.78:43128) 2024-09-21T09:20:38.718 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.713+0000 7f4e63fff640 1 -- 172.21.15.78:0/529245718 learned_addr learned my addr 172.21.15.78:0/529245718 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:20:38.718 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.713+0000 7f4e63fff640 1 -- 172.21.15.78:0/529245718 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f4e64091f10 msgr2=0x7f4e6408ed90 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-21T09:20:38.718 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.713+0000 7f4e63fff640 1 --2- 172.21.15.78:0/529245718 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f4e64091f10 0x7f4e6408ed90 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.718 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.713+0000 7f4e63fff640 1 -- 172.21.15.78:0/529245718 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f4e64088ac0 msgr2=0x7f4e6408e310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:38.718 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.713+0000 7f4e63fff640 1 --2- 172.21.15.78:0/529245718 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f4e64088ac0 0x7f4e6408e310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.719 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.713+0000 7f4e63fff640 1 -- 172.21.15.78:0/529245718 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4e5c009900 con 0x7f4e6408b410 2024-09-21T09:20:38.719 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.714+0000 7f4e63fff640 1 --2- 172.21.15.78:0/529245718 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f4e6408b410 0x7f4e6408e850 secure :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0x7f4e5c009e50 tx=0x7f4e5c02c720 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:20:38.719 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.716+0000 7f4647fff640 1 -- 172.21.15.78:0/3561683452 <== mon.1 v2:172.21.15.78:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+73245 (secure 0 0 0) 0x7f4660130cb0 con 0x7f466012a500 2024-09-21T09:20:38.720 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.717+0000 7f06470d7640 1 -- 172.21.15.78:0/804017614 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f063800dbf0 msgr2=0x7f06380d8c10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:38.720 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.717+0000 7f06470d7640 1 --2- 172.21.15.78:0/804017614 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f063800dbf0 0x7f06380d8c10 secure :-1 s=READY pgs=20 cs=0 l=1 rev1=1 crypto rx=0x7f062c005fd0 tx=0x7f062c02a480 comp rx=0 tx=0).stop 2024-09-21T09:20:38.720 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.717+0000 7f06470d7640 1 -- 172.21.15.78:0/804017614 shutdown_connections 2024-09-21T09:20:38.720 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.717+0000 7f06470d7640 1 --2- 172.21.15.78:0/804017614 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f063800dbf0 0x7f06380d8c10 unknown :-1 s=CLOSED pgs=20 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.720 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.717+0000 7f06470d7640 1 --2- 172.21.15.78:0/804017614 >> v2:172.21.15.78:3300/0 conn(0x7f063800cea0 0x7f063800d280 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.720 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.717+0000 7f06470d7640 1 -- 172.21.15.78:0/804017614 >> 172.21.15.78:0/804017614 conn(0x7f06380059e0 msgr2=0x7f0638005df0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:20:38.721 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.717+0000 7f06470d7640 1 -- 172.21.15.78:0/804017614 shutdown_connections 2024-09-21T09:20:38.721 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.717+0000 7f06470d7640 1 -- 172.21.15.78:0/804017614 wait complete. 2024-09-21T09:20:38.721 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.718+0000 7f06470d7640 1 Processor -- start 2024-09-21T09:20:38.721 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.719+0000 7f06470d7640 1 -- start start 2024-09-21T09:20:38.724 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.722+0000 7f1aa7fff640 1 -- 172.21.15.78:0/3145895023 <== mon.1 v2:172.21.15.78:3301/0 11 ==== mon_command_ack([{prefix=config set, name=debug_mds_balancer}]=0 v8)=0 v8) v1 ==== 117+0+0 (secure 0 0 0) 0x7f1ab8054020 con 0x7f1ac006bbc0 2024-09-21T09:20:38.725 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.719+0000 7f06470d7640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f063800cea0 0x7f0638017230 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:38.725 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.719+0000 7f06470d7640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f063800d7c0 0x7f0638017770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:38.725 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.719+0000 7f06470d7640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f06381778f0 0x7f0638177d70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:38.725 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.719+0000 7f06470d7640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0638019fd0 con 0x7f063800cea0 2024-09-21T09:20:38.725 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.719+0000 7f06470d7640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0638018390 con 0x7f063800d7c0 2024-09-21T09:20:38.725 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.719+0000 7f06470d7640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 v1 -- 0x7f063801a570 con 0x7f06381778f0 2024-09-21T09:20:38.726 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.722+0000 7f063ffff640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f063800d7c0 0x7f0638017770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:38.726 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.722+0000 7f063ffff640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f063800d7c0 0x7f0638017770 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3301/0 says I am v2:172.21.15.78:50192/0 (socket says 172.21.15.78:50192) 2024-09-21T09:20:38.726 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.722+0000 7f063ffff640 1 -- 172.21.15.78:0/4268584608 learned_addr learned my addr 172.21.15.78:0/4268584608 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:20:38.726 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.722+0000 7f0645674640 1 --2- 172.21.15.78:0/4268584608 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f06381778f0 0x7f0638177d70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:38.726 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.717+0000 7f4e617fa640 1 -- 172.21.15.78:0/529245718 <== mon.2 v2:172.21.15.78:3302/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4e5c03e6f0 con 0x7f4e6408b410 2024-09-21T09:20:38.726 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.717+0000 7f4e6acc5640 1 -- 172.21.15.78:0/529245718 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4e6408e080 con 0x7f4e6408b410 2024-09-21T09:20:38.727 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.717+0000 7f4e6acc5640 1 -- 172.21.15.78:0/529245718 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4e6408fb50 con 0x7f4e6408b410 2024-09-21T09:20:38.727 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.723+0000 7f4e617fa640 1 -- 172.21.15.78:0/529245718 <== mon.2 v2:172.21.15.78:3302/0 2 ==== config(4 keys) v1 ==== 110+0+0 (secure 0 0 0) 0x7f4e5c03e850 con 0x7f4e6408b410 2024-09-21T09:20:38.727 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.723+0000 7f4e617fa640 1 -- 172.21.15.78:0/529245718 <== mon.2 v2:172.21.15.78:3302/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4e5c03ebc0 con 0x7f4e6408b410 2024-09-21T09:20:38.727 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.723+0000 7f4e617fa640 1 -- 172.21.15.78:0/529245718 <== mon.2 v2:172.21.15.78:3302/0 4 ==== mgrmap(e 1) v1 ==== 668+0+0 (secure 0 0 0) 0x7f4e5c00a040 con 0x7f4e6408b410 2024-09-21T09:20:38.727 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.723+0000 7f4e617fa640 1 -- 172.21.15.78:0/529245718 <== mon.2 v2:172.21.15.78:3302/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 725+0+0 (secure 0 0 0) 0x7f4e5c030070 con 0x7f4e6408b410 2024-09-21T09:20:38.727 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.723+0000 7f063ffff640 1 -- 172.21.15.78:0/4268584608 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f06381778f0 msgr2=0x7f0638177d70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:38.727 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.723+0000 7f063ffff640 1 --2- 172.21.15.78:0/4268584608 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f06381778f0 0x7f0638177d70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.727 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.723+0000 7f063ffff640 1 -- 172.21.15.78:0/4268584608 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f063800cea0 msgr2=0x7f0638017230 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:38.727 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.723+0000 7f063ffff640 1 --2- 172.21.15.78:0/4268584608 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f063800cea0 0x7f0638017230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.728 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.723+0000 7f063ffff640 1 -- 172.21.15.78:0/4268584608 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f062c009560 con 0x7f063800d7c0 2024-09-21T09:20:38.728 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.723+0000 7f063ffff640 1 --2- 172.21.15.78:0/4268584608 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f063800d7c0 0x7f0638017770 secure :-1 s=READY pgs=21 cs=0 l=1 rev1=1 crypto rx=0x7f064004efc0 tx=0x7f0640073830 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:20:38.728 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.724+0000 7f063d7fa640 1 -- 172.21.15.78:0/4268584608 <== mon.1 v2:172.21.15.78:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0640073dd0 con 0x7f063800d7c0 2024-09-21T09:20:38.728 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.724+0000 7f063d7fa640 1 -- 172.21.15.78:0/4268584608 <== mon.1 v2:172.21.15.78:3301/0 2 ==== config(4 keys) v1 ==== 110+0+0 (secure 0 0 0) 0x7f0640065bd0 con 0x7f063800d7c0 2024-09-21T09:20:38.728 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.724+0000 7f063d7fa640 1 -- 172.21.15.78:0/4268584608 <== mon.1 v2:172.21.15.78:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0640065fc0 con 0x7f063800d7c0 2024-09-21T09:20:38.728 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.724+0000 7f06470d7640 1 -- 172.21.15.78:0/4268584608 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0638006b30 con 0x7f063800d7c0 2024-09-21T09:20:38.728 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.725+0000 7f06470d7640 1 -- 172.21.15.78:0/4268584608 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0638018aa0 con 0x7f063800d7c0 2024-09-21T09:20:38.728 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.725+0000 7f063d7fa640 1 -- 172.21.15.78:0/4268584608 <== mon.1 v2:172.21.15.78:3301/0 4 ==== mgrmap(e 1) v1 ==== 668+0+0 (secure 0 0 0) 0x7f064006dd50 con 0x7f063800d7c0 2024-09-21T09:20:38.729 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.725+0000 7f063d7fa640 1 -- 172.21.15.78:0/4268584608 <== mon.1 v2:172.21.15.78:3301/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 725+0+0 (secure 0 0 0) 0x7f0640069e40 con 0x7f063800d7c0 2024-09-21T09:20:38.729 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.725+0000 7f4e6acc5640 1 -- 172.21.15.78:0/529245718 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4e44005250 con 0x7f4e6408b410 2024-09-21T09:20:38.729 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.726+0000 7f7cd22ec640 1 -- 172.21.15.78:0/3131609792 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f7cc40b2a90 msgr2=0x7f7cc40b3900 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:38.730 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.726+0000 7f7cd22ec640 1 --2- 172.21.15.78:0/3131609792 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f7cc40b2a90 0x7f7cc40b3900 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7f7cc0005fd0 tx=0x7f7cc003b500 comp rx=0 tx=0).stop 2024-09-21T09:20:38.730 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.726+0000 7f7cd22ec640 1 -- 172.21.15.78:0/3131609792 shutdown_connections 2024-09-21T09:20:38.730 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.726+0000 7f7cd22ec640 1 --2- 172.21.15.78:0/3131609792 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f7cc40b2a90 0x7f7cc40b3900 unknown :-1 s=CLOSED pgs=19 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.730 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.726+0000 7f7cd22ec640 1 --2- 172.21.15.78:0/3131609792 >> v2:172.21.15.78:3300/0 conn(0x7f7cc401a790 0x7f7cc40b2550 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.730 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.726+0000 7f7cd22ec640 1 -- 172.21.15.78:0/3131609792 >> 172.21.15.78:0/3131609792 conn(0x7f7cc4006640 msgr2=0x7f7cc4006a50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:20:38.732 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.727+0000 7f1ac4f9d640 1 -- 172.21.15.78:0/3145895023 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f1ac006bbc0 msgr2=0x7f1ac008b860 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:38.732 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.727+0000 7f1ac4f9d640 1 --2- 172.21.15.78:0/3145895023 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f1ac006bbc0 0x7f1ac008b860 secure :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0x7f1ac0073d30 tx=0x7f1ab8004a40 comp rx=0 tx=0).stop 2024-09-21T09:20:38.732 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.727+0000 7f1ac4f9d640 1 -- 172.21.15.78:0/3145895023 shutdown_connections 2024-09-21T09:20:38.732 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.727+0000 7f1ac4f9d640 1 --2- 172.21.15.78:0/3145895023 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f1ac012b9c0 0x7f1ac008c2e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.733 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.727+0000 7f1ac4f9d640 1 --2- 172.21.15.78:0/3145895023 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f1ac0071680 0x7f1ac008bda0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.733 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.727+0000 7f1ac4f9d640 1 --2- 172.21.15.78:0/3145895023 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f1ac006bbc0 0x7f1ac008b860 secure :-1 s=CLOSED pgs=12 cs=0 l=1 rev1=1 crypto rx=0x7f1ac0073d30 tx=0x7f1ab8004a40 comp rx=0 tx=0).stop 2024-09-21T09:20:38.733 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.727+0000 7f1ac4f9d640 1 -- 172.21.15.78:0/3145895023 >> 172.21.15.78:0/3145895023 conn(0x7f1ac009b740 msgr2=0x7f1ac00909d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:20:38.733 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.730+0000 7f1ac4f9d640 1 -- 172.21.15.78:0/3145895023 shutdown_connections 2024-09-21T09:20:38.733 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.730+0000 7f1ac4f9d640 1 -- 172.21.15.78:0/3145895023 wait complete. 2024-09-21T09:20:38.733 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.730+0000 7f4e617fa640 1 -- 172.21.15.78:0/529245718 <== mon.2 v2:172.21.15.78:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+73245 (secure 0 0 0) 0x7f4e5c04e060 con 0x7f4e6408b410 2024-09-21T09:20:38.733 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.730+0000 7f06470d7640 1 -- 172.21.15.78:0/4268584608 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f063800dbf0 con 0x7f063800d7c0 2024-09-21T09:20:38.740 INFO:tasks.ceph.mgr.x.smithi078.stderr:2024-09-21T09:20:38.736+0000 7fda56037040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-21T09:20:38.740 INFO:tasks.ceph.mgr.y.smithi078.stderr:2024-09-21T09:20:38.735+0000 7ff872721040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-21T09:20:38.740 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.734+0000 7f7cd22ec640 1 -- 172.21.15.78:0/3131609792 shutdown_connections 2024-09-21T09:20:38.740 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.734+0000 7f7cd22ec640 1 -- 172.21.15.78:0/3131609792 wait complete. 2024-09-21T09:20:38.741 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.735+0000 7f7cd22ec640 1 Processor -- start 2024-09-21T09:20:38.741 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.735+0000 7f7cd22ec640 1 -- start start 2024-09-21T09:20:38.741 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.736+0000 7f7cd22ec640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f7cc401a790 0x7f7cc40ab5a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:38.741 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.736+0000 7f7cd22ec640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f7cc401b2f0 0x7f7cc40abae0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:38.741 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.736+0000 7f7cd22ec640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f7cc40b2a90 0x7f7cc40ac020 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:38.741 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.736+0000 7f7cd22ec640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7cc40ad050 con 0x7f7cc401b2f0 2024-09-21T09:20:38.741 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.736+0000 7f7cd22ec640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7cc40aed50 con 0x7f7cc40b2a90 2024-09-21T09:20:38.741 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.736+0000 7f7cd22ec640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 v1 -- 0x7f7cc40ae1a0 con 0x7f7cc401a790 2024-09-21T09:20:38.741 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.736+0000 7f7ccbfff640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f7cc401a790 0x7f7cc40ab5a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:38.741 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.736+0000 7f7ccbfff640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f7cc401a790 0x7f7cc40ab5a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3302/0 says I am v2:172.21.15.78:43142/0 (socket says 172.21.15.78:43142) 2024-09-21T09:20:38.741 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.736+0000 7f7ccbfff640 1 -- 172.21.15.78:0/2828725089 learned_addr learned my addr 172.21.15.78:0/2828725089 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:20:38.741 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.737+0000 7f7cd0889640 1 --2- 172.21.15.78:0/2828725089 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f7cc40b2a90 0x7f7cc40ac020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:38.741 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.737+0000 7f7ccbfff640 1 -- 172.21.15.78:0/2828725089 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f7cc40b2a90 msgr2=0x7f7cc40ac020 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:38.742 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.737+0000 7f7ccbfff640 1 --2- 172.21.15.78:0/2828725089 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f7cc40b2a90 0x7f7cc40ac020 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.742 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.737+0000 7f7ccbfff640 1 -- 172.21.15.78:0/2828725089 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f7cc401b2f0 msgr2=0x7f7cc40abae0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:38.742 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.737+0000 7f7ccb7fe640 1 --2- 172.21.15.78:0/2828725089 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f7cc401b2f0 0x7f7cc40abae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:38.742 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.740+0000 7f82b7fff640 1 -- 172.21.15.78:0/1258504649 <== mon.1 v2:172.21.15.78:3301/0 8 ==== mon_command_ack([{prefix=config set, name=mds_debug_frag}]=0 v9)=0 v9) v1 ==== 115+0+0 (secure 0 0 0) 0x7f82c000ecf0 con 0x7f82d0085cb0 2024-09-21T09:20:38.743 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.740+0000 7f7ccbfff640 1 --2- 172.21.15.78:0/2828725089 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f7cc401b2f0 0x7f7cc40abae0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.743 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.740+0000 7f7ccbfff640 1 -- 172.21.15.78:0/2828725089 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7cc0009560 con 0x7f7cc401a790 2024-09-21T09:20:38.743 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.741+0000 7f7cd0889640 1 --2- 172.21.15.78:0/2828725089 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f7cc40b2a90 0x7f7cc40ac020 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-09-21T09:20:38.743 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.741+0000 7f7ccb7fe640 1 --2- 172.21.15.78:0/2828725089 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f7cc401b2f0 0x7f7cc40abae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-09-21T09:20:38.745 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.742+0000 7f82d5b76640 1 -- 172.21.15.78:0/1258504649 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f82d0085cb0 msgr2=0x7f82d0087ae0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:38.745 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.742+0000 7f82d5b76640 1 --2- 172.21.15.78:0/1258504649 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f82d0085cb0 0x7f82d0087ae0 secure :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0x7f82c000b590 tx=0x7f82c000ba50 comp rx=0 tx=0).stop 2024-09-21T09:20:38.745 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.742+0000 7f82d5b76640 1 -- 172.21.15.78:0/1258504649 shutdown_connections 2024-09-21T09:20:38.745 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.742+0000 7f82d5b76640 1 --2- 172.21.15.78:0/1258504649 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f82d008b300 0x7f82d008b7b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.745 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.742+0000 7f82d5b76640 1 --2- 172.21.15.78:0/1258504649 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f82d0091f10 0x7f82d0088020 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.745 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.742+0000 7f82d5b76640 1 --2- 172.21.15.78:0/1258504649 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f82d0085cb0 0x7f82d0087ae0 unknown :-1 s=CLOSED pgs=14 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.745 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.742+0000 7f82d5b76640 1 -- 172.21.15.78:0/1258504649 >> 172.21.15.78:0/1258504649 conn(0x7f82d00986a0 msgr2=0x7f82d0080460 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:20:38.745 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.742+0000 7f82d5b76640 1 -- 172.21.15.78:0/1258504649 shutdown_connections 2024-09-21T09:20:38.745 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.742+0000 7f82d5b76640 1 -- 172.21.15.78:0/1258504649 wait complete. 2024-09-21T09:20:38.746 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.744+0000 7f7ccbfff640 1 --2- 172.21.15.78:0/2828725089 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f7cc401a790 0x7f7cc40ab5a0 secure :-1 s=READY pgs=13 cs=0 l=1 rev1=1 crypto rx=0x7f7ccc069c80 tx=0x7f7ccc066d10 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:20:38.747 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.745+0000 7f7cc8ff9640 1 -- 172.21.15.78:0/2828725089 <== mon.2 v2:172.21.15.78:3302/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7ccc0008d0 con 0x7f7cc401a790 2024-09-21T09:20:38.750 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.745+0000 7f7cd22ec640 1 -- 172.21.15.78:0/2828725089 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7cc40ad960 con 0x7f7cc401a790 2024-09-21T09:20:38.750 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.745+0000 7f7cd22ec640 1 -- 172.21.15.78:0/2828725089 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7cc40af010 con 0x7f7cc401a790 2024-09-21T09:20:38.750 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.746+0000 7f7cc8ff9640 1 -- 172.21.15.78:0/2828725089 <== mon.2 v2:172.21.15.78:3302/0 2 ==== config(4 keys) v1 ==== 110+0+0 (secure 0 0 0) 0x7f7ccc073bb0 con 0x7f7cc401a790 2024-09-21T09:20:38.750 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.746+0000 7f7cc8ff9640 1 -- 172.21.15.78:0/2828725089 <== mon.2 v2:172.21.15.78:3302/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7ccc073d20 con 0x7f7cc401a790 2024-09-21T09:20:38.750 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.747+0000 7f7cc8ff9640 1 -- 172.21.15.78:0/2828725089 <== mon.2 v2:172.21.15.78:3302/0 4 ==== mgrmap(e 1) v1 ==== 668+0+0 (secure 0 0 0) 0x7f7ccc065ed0 con 0x7f7cc401a790 2024-09-21T09:20:38.750 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.747+0000 7f7cc8ff9640 1 -- 172.21.15.78:0/2828725089 <== mon.2 v2:172.21.15.78:3302/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 725+0+0 (secure 0 0 0) 0x7f7ccc069e40 con 0x7f7cc401a790 2024-09-21T09:20:38.751 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.749+0000 7f7cd22ec640 1 -- 172.21.15.78:0/2828725089 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7cac005250 con 0x7f7cc401a790 2024-09-21T09:20:38.759 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.757+0000 7f7cc8ff9640 1 -- 172.21.15.78:0/2828725089 <== mon.2 v2:172.21.15.78:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+73245 (secure 0 0 0) 0x7f7ccc097060 con 0x7f7cc401a790 2024-09-21T09:20:38.769 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.734+0000 7f063d7fa640 1 -- 172.21.15.78:0/4268584608 <== mon.1 v2:172.21.15.78:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+73245 (secure 0 0 0) 0x7f0640085060 con 0x7f063800d7c0 2024-09-21T09:20:38.810 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.802+0000 7f01b6e2b640 1 -- 172.21.15.78:0/1257577644 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command([{prefix=config set, name=mds_debug_scatterstat}] v 0) v1 -- 0x7f01900057d0 con 0x7f01a80b4e80 2024-09-21T09:20:38.812 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.805+0000 7f4190cc9640 1 -- 172.21.15.78:0/480829784 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f418c088fb0 msgr2=0x7f418c089390 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:38.812 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.806+0000 7f4190cc9640 1 --2- 172.21.15.78:0/480829784 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f418c088fb0 0x7f418c089390 secure :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0x7f417c009830 tx=0x7f417c02a360 comp rx=0 tx=0).stop 2024-09-21T09:20:38.812 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.808+0000 7f4190cc9640 1 -- 172.21.15.78:0/480829784 shutdown_connections 2024-09-21T09:20:38.812 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.808+0000 7f4190cc9640 1 --2- 172.21.15.78:0/480829784 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f418c135010 0x7f418c137400 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.812 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.808+0000 7f4190cc9640 1 --2- 172.21.15.78:0/480829784 >> v2:172.21.15.78:3300/0 conn(0x7f418c0898d0 0x7f418c089cb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.812 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.808+0000 7f4190cc9640 1 --2- 172.21.15.78:0/480829784 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f418c088fb0 0x7f418c089390 unknown :-1 s=CLOSED pgs=14 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.812 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.808+0000 7f4190cc9640 1 -- 172.21.15.78:0/480829784 >> 172.21.15.78:0/480829784 conn(0x7f418c098570 msgr2=0x7f418c098980 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:20:38.815 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.810+0000 7f4190cc9640 1 -- 172.21.15.78:0/480829784 shutdown_connections 2024-09-21T09:20:38.815 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.810+0000 7f4190cc9640 1 -- 172.21.15.78:0/480829784 wait complete. 2024-09-21T09:20:38.815 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.811+0000 7f4190cc9640 1 Processor -- start 2024-09-21T09:20:38.815 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.813+0000 7f01ad7fa640 1 -- 172.21.15.78:0/1257577644 <== mon.0 v2:172.21.15.78:3300/0 7 ==== mon_command_ack([{prefix=config set, name=mds_debug_scatterstat}]=0 v10)=0 v10) v1 ==== 122+0+0 (secure 0 0 0) 0x7f01a4040070 con 0x7f01a80b4e80 2024-09-21T09:20:38.817 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.812+0000 7f4190cc9640 1 -- start start 2024-09-21T09:20:38.817 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.812+0000 7f4190cc9640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f418c088fb0 0x7f418c12bc30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:38.817 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.812+0000 7f4190cc9640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f418c0898d0 0x7f418c12c170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:38.817 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.812+0000 7f4190cc9640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f418c135010 0x7f418c08b0c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:38.817 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.812+0000 7f4190cc9640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f418c08d4b0 con 0x7f418c088fb0 2024-09-21T09:20:38.818 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.812+0000 7f4190cc9640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f418c133c30 con 0x7f418c135010 2024-09-21T09:20:38.818 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.812+0000 7f4190cc9640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 v1 -- 0x7f418c133370 con 0x7f418c0898d0 2024-09-21T09:20:38.818 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.813+0000 7f4189d9b640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f418c0898d0 0x7f418c12c170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:38.821 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.817+0000 7f01b6e2b640 1 -- 172.21.15.78:0/1257577644 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f01a80b4e80 msgr2=0x7f01a8144cb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:38.821 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.817+0000 7f01b6e2b640 1 --2- 172.21.15.78:0/1257577644 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f01a80b4e80 0x7f01a8144cb0 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7f01a8007d90 tx=0x7f01a4004a50 comp rx=0 tx=0).stop 2024-09-21T09:20:38.821 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.819+0000 7f01b6e2b640 1 -- 172.21.15.78:0/1257577644 shutdown_connections 2024-09-21T09:20:38.821 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.819+0000 7f01b6e2b640 1 --2- 172.21.15.78:0/1257577644 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f01a80b4e80 0x7f01a8144cb0 unknown :-1 s=CLOSED pgs=19 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.821 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.819+0000 7f01b6e2b640 1 --2- 172.21.15.78:0/1257577644 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f01a8019370 0x7f01a8144770 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.821 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.819+0000 7f01b6e2b640 1 --2- 172.21.15.78:0/1257577644 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f01a8018880 0x7f01a814dff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.821 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.819+0000 7f01b6e2b640 1 -- 172.21.15.78:0/1257577644 >> 172.21.15.78:0/1257577644 conn(0x7f01a8005bf0 msgr2=0x7f01a801b760 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:20:38.822 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.813+0000 7f4189d9b640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f418c0898d0 0x7f418c12c170 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3302/0 says I am v2:172.21.15.78:43166/0 (socket says 172.21.15.78:43166) 2024-09-21T09:20:38.822 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.813+0000 7f4189d9b640 1 -- 172.21.15.78:0/2752674021 learned_addr learned my addr 172.21.15.78:0/2752674021 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:20:38.822 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.814+0000 7f4189d9b640 1 -- 172.21.15.78:0/2752674021 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f418c135010 msgr2=0x7f418c08b0c0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-21T09:20:38.822 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.819+0000 7f01b6e2b640 1 -- 172.21.15.78:0/1257577644 shutdown_connections 2024-09-21T09:20:38.823 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.814+0000 7f4189d9b640 1 --2- 172.21.15.78:0/2752674021 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f418c135010 0x7f418c08b0c0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.823 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.814+0000 7f4189d9b640 1 -- 172.21.15.78:0/2752674021 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f418c088fb0 msgr2=0x7f418c12bc30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:38.823 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.814+0000 7f4189d9b640 1 --2- 172.21.15.78:0/2752674021 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f418c088fb0 0x7f418c12bc30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.823 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.814+0000 7f4189d9b640 1 -- 172.21.15.78:0/2752674021 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f417c009490 con 0x7f418c0898d0 2024-09-21T09:20:38.823 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.814+0000 7f4189d9b640 1 --2- 172.21.15.78:0/2752674021 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f418c0898d0 0x7f418c12c170 secure :-1 s=READY pgs=15 cs=0 l=1 rev1=1 crypto rx=0x7f418401afc0 tx=0x7f418401b790 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:20:38.824 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.815+0000 7f417affd640 1 -- 172.21.15.78:0/2752674021 <== mon.2 v2:172.21.15.78:3302/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4184004b20 con 0x7f418c0898d0 2024-09-21T09:20:38.825 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.815+0000 7f417affd640 1 -- 172.21.15.78:0/2752674021 <== mon.2 v2:172.21.15.78:3302/0 2 ==== config(4 keys) v1 ==== 110+0+0 (secure 0 0 0) 0x7f4184004d50 con 0x7f418c0898d0 2024-09-21T09:20:38.825 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.815+0000 7f417affd640 1 -- 172.21.15.78:0/2752674021 <== mon.2 v2:172.21.15.78:3302/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4184005260 con 0x7f418c0898d0 2024-09-21T09:20:38.825 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.815+0000 7f4190cc9640 1 -- 172.21.15.78:0/2752674021 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f418c0712c0 con 0x7f418c0898d0 2024-09-21T09:20:38.825 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.823+0000 7f01b6e2b640 1 -- 172.21.15.78:0/1257577644 wait complete. 2024-09-21T09:20:38.826 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.815+0000 7f4190cc9640 1 -- 172.21.15.78:0/2752674021 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f418c08cc50 con 0x7f418c0898d0 2024-09-21T09:20:38.826 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.818+0000 7f4190cc9640 1 -- 172.21.15.78:0/2752674021 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f418c089390 con 0x7f418c0898d0 2024-09-21T09:20:38.826 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.819+0000 7f417affd640 1 -- 172.21.15.78:0/2752674021 <== mon.2 v2:172.21.15.78:3302/0 4 ==== mgrmap(e 1) v1 ==== 668+0+0 (secure 0 0 0) 0x7f41840053c0 con 0x7f418c0898d0 2024-09-21T09:20:38.826 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.819+0000 7f417affd640 1 -- 172.21.15.78:0/2752674021 <== mon.2 v2:172.21.15.78:3302/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 725+0+0 (secure 0 0 0) 0x7f418400f6b0 con 0x7f418c0898d0 2024-09-21T09:20:38.826 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.823+0000 7f417affd640 1 -- 172.21.15.78:0/2752674021 <== mon.2 v2:172.21.15.78:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+73245 (secure 0 0 0) 0x7f4184023060 con 0x7f418c0898d0 2024-09-21T09:20:38.834 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.832+0000 7f7283653640 1 -- 172.21.15.78:0/3393305199 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f727c088fb0 msgr2=0x7f727c089390 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:38.834 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.832+0000 7f7283653640 1 --2- 172.21.15.78:0/3393305199 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f727c088fb0 0x7f727c089390 secure :-1 s=READY pgs=22 cs=0 l=1 rev1=1 crypto rx=0x7f726c005fd0 tx=0x7f726c02a4f0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.835 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.833+0000 7f7283653640 1 -- 172.21.15.78:0/3393305199 shutdown_connections 2024-09-21T09:20:38.835 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.833+0000 7f7283653640 1 --2- 172.21.15.78:0/3393305199 >> v2:172.21.15.78:3300/0 conn(0x7f727c0898d0 0x7f727c089cb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.835 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.833+0000 7f7283653640 1 --2- 172.21.15.78:0/3393305199 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f727c088fb0 0x7f727c089390 unknown :-1 s=CLOSED pgs=22 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.835 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.833+0000 7f7283653640 1 -- 172.21.15.78:0/3393305199 >> 172.21.15.78:0/3393305199 conn(0x7f727c098570 msgr2=0x7f727c098980 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:20:38.835 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.833+0000 7f7283653640 1 -- 172.21.15.78:0/3393305199 shutdown_connections 2024-09-21T09:20:38.836 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.833+0000 7f7283653640 1 -- 172.21.15.78:0/3393305199 wait complete. 2024-09-21T09:20:38.836 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.834+0000 7f7283653640 1 Processor -- start 2024-09-21T09:20:38.837 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.834+0000 7f7283653640 1 -- start start 2024-09-21T09:20:38.838 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.835+0000 7f7283653640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f727c088fb0 0x7f727c12b350 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:38.838 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.835+0000 7f7283653640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f727c0898d0 0x7f727c12b890 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:38.838 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.835+0000 7f7283653640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f727c135010 0x7f727c12bdd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:38.838 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.835+0000 7f7283653640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f727c1214b0 con 0x7f727c135010 2024-09-21T09:20:38.838 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.835+0000 7f7283653640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f727c1217c0 con 0x7f727c088fb0 2024-09-21T09:20:38.838 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.835+0000 7f7283653640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 v1 -- 0x7f727c11fac0 con 0x7f727c0898d0 2024-09-21T09:20:38.839 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.835+0000 7f7280bee640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f727c0898d0 0x7f727c12b890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:38.839 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.836+0000 7f7280bee640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f727c0898d0 0x7f727c12b890 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3302/0 says I am v2:172.21.15.78:43182/0 (socket says 172.21.15.78:43182) 2024-09-21T09:20:38.839 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.836+0000 7f7280bee640 1 -- 172.21.15.78:0/374465910 learned_addr learned my addr 172.21.15.78:0/374465910 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:20:38.839 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.836+0000 7f7280bee640 1 -- 172.21.15.78:0/374465910 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f727c088fb0 msgr2=0x7f727c12b350 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-21T09:20:38.839 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.836+0000 7f72813ef640 1 --2- 172.21.15.78:0/374465910 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f727c088fb0 0x7f727c12b350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:38.839 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.836+0000 7f7280bee640 1 --2- 172.21.15.78:0/374465910 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f727c088fb0 0x7f727c12b350 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.839 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.836+0000 7f7280bee640 1 -- 172.21.15.78:0/374465910 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f727c135010 msgr2=0x7f727c12bdd0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-21T09:20:38.840 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.836+0000 7f7280bee640 1 --2- 172.21.15.78:0/374465910 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f727c135010 0x7f727c12bdd0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.840 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.836+0000 7f7280bee640 1 -- 172.21.15.78:0/374465910 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f726c009560 con 0x7f727c0898d0 2024-09-21T09:20:38.840 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.837+0000 7f7280bee640 1 --2- 172.21.15.78:0/374465910 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f727c0898d0 0x7f727c12b890 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7f7274005fd0 tx=0x7f727400a620 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:20:38.840 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.838+0000 7f7271ffb640 1 -- 172.21.15.78:0/374465910 <== mon.2 v2:172.21.15.78:3302/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f727400f650 con 0x7f727c0898d0 2024-09-21T09:20:38.842 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.838+0000 7f72813ef640 1 --2- 172.21.15.78:0/374465910 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f727c088fb0 0x7f727c12b350 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-09-21T09:20:38.843 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.838+0000 7f7271ffb640 1 -- 172.21.15.78:0/374465910 <== mon.2 v2:172.21.15.78:3302/0 2 ==== config(4 keys) v1 ==== 110+0+0 (secure 0 0 0) 0x7f727400f880 con 0x7f727c0898d0 2024-09-21T09:20:38.843 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.838+0000 7f7271ffb640 1 -- 172.21.15.78:0/374465910 <== mon.2 v2:172.21.15.78:3302/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f727400fc50 con 0x7f727c0898d0 2024-09-21T09:20:38.843 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.838+0000 7f7283653640 1 -- 172.21.15.78:0/374465910 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f727c12c310 con 0x7f727c0898d0 2024-09-21T09:20:38.843 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.838+0000 7f7283653640 1 -- 172.21.15.78:0/374465910 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f727c120c10 con 0x7f727c0898d0 2024-09-21T09:20:38.843 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.839+0000 7f7271ffb640 1 -- 172.21.15.78:0/374465910 <== mon.2 v2:172.21.15.78:3302/0 4 ==== mgrmap(e 1) v1 ==== 668+0+0 (secure 0 0 0) 0x7f7274018400 con 0x7f727c0898d0 2024-09-21T09:20:38.843 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.839+0000 7f7271ffb640 1 -- 172.21.15.78:0/374465910 <== mon.2 v2:172.21.15.78:3302/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 725+0+0 (secure 0 0 0) 0x7f7274022730 con 0x7f727c0898d0 2024-09-21T09:20:38.843 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.840+0000 7f7283653640 1 -- 172.21.15.78:0/374465910 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f727c120ec0 con 0x7f727c0898d0 2024-09-21T09:20:38.847 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.844+0000 7f7271ffb640 1 -- 172.21.15.78:0/374465910 <== mon.2 v2:172.21.15.78:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+73245 (secure 0 0 0) 0x7f727c120ec0 con 0x7f727c0898d0 2024-09-21T09:20:38.882 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.879+0000 7efe34be6640 1 -- 172.21.15.78:0/2163523916 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7efe28012000 msgr2=0x7efe280b9e40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:38.882 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.879+0000 7efe34be6640 1 --2- 172.21.15.78:0/2163523916 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7efe28012000 0x7efe280b9e40 secure :-1 s=READY pgs=17 cs=0 l=1 rev1=1 crypto rx=0x7efe1c005fd0 tx=0x7efe1c02a420 comp rx=0 tx=0).stop 2024-09-21T09:20:38.883 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.880+0000 7fd0ce3fb640 1 -- 172.21.15.78:0/3042191618 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fd0c808a1f0 msgr2=0x7fd0c8088fb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:38.883 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.880+0000 7fd0ce3fb640 1 --2- 172.21.15.78:0/3042191618 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fd0c808a1f0 0x7fd0c8088fb0 secure :-1 s=READY pgs=23 cs=0 l=1 rev1=1 crypto rx=0x7fd0b8009880 tx=0x7fd0b802a440 comp rx=0 tx=0).stop 2024-09-21T09:20:38.883 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.880+0000 7fd0ce3fb640 1 -- 172.21.15.78:0/3042191618 shutdown_connections 2024-09-21T09:20:38.883 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.880+0000 7fd0ce3fb640 1 --2- 172.21.15.78:0/3042191618 >> v2:172.21.15.78:3300/0 conn(0x7fd0c8135fb0 0x7fd0c81383a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.883 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.880+0000 7fd0ce3fb640 1 --2- 172.21.15.78:0/3042191618 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fd0c808a1f0 0x7fd0c8088fb0 unknown :-1 s=CLOSED pgs=23 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.883 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.880+0000 7fd0ce3fb640 1 -- 172.21.15.78:0/3042191618 >> 172.21.15.78:0/3042191618 conn(0x7fd0c809a7e0 msgr2=0x7fd0c809abf0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:20:38.883 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.880+0000 7fd0ce3fb640 1 -- 172.21.15.78:0/3042191618 shutdown_connections 2024-09-21T09:20:38.883 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.880+0000 7fd0ce3fb640 1 -- 172.21.15.78:0/3042191618 wait complete. 2024-09-21T09:20:38.885 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.881+0000 7efe34be6640 1 -- 172.21.15.78:0/2163523916 shutdown_connections 2024-09-21T09:20:38.885 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.881+0000 7efe34be6640 1 --2- 172.21.15.78:0/2163523916 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7efe28012000 0x7efe280b9e40 unknown :-1 s=CLOSED pgs=17 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.885 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.881+0000 7efe34be6640 1 --2- 172.21.15.78:0/2163523916 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7efe28011be0 0x7efe280b78f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.886 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.881+0000 7efe34be6640 1 --2- 172.21.15.78:0/2163523916 >> v2:172.21.15.78:3300/0 conn(0x7efe280112c0 0x7efe280116a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.886 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.881+0000 7efe34be6640 1 -- 172.21.15.78:0/2163523916 >> 172.21.15.78:0/2163523916 conn(0x7efe28005cf0 msgr2=0x7efe28006100 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:20:38.886 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.881+0000 7efe34be6640 1 -- 172.21.15.78:0/2163523916 shutdown_connections 2024-09-21T09:20:38.886 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.881+0000 7efe34be6640 1 -- 172.21.15.78:0/2163523916 wait complete. 2024-09-21T09:20:38.886 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.882+0000 7efe34be6640 1 Processor -- start 2024-09-21T09:20:38.886 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.881+0000 7fd0ce3fb640 1 Processor -- start 2024-09-21T09:20:38.887 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.883+0000 7efe34be6640 1 -- start start 2024-09-21T09:20:38.887 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.883+0000 7efe34be6640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7efe280112c0 0x7efe280b4da0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:38.887 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.883+0000 7efe34be6640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7efe28011be0 0x7efe280b52e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:38.887 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.884+0000 7efe2f7fe640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7efe280112c0 0x7efe280b4da0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:38.887 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.884+0000 7efe2f7fe640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7efe280112c0 0x7efe280b4da0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3301/0 says I am v2:172.21.15.78:50292/0 (socket says 172.21.15.78:50292) 2024-09-21T09:20:38.887 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.884+0000 7efe2effd640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7efe28011be0 0x7efe280b52e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:38.888 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.882+0000 7fd0ce3fb640 1 -- start start 2024-09-21T09:20:38.888 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.882+0000 7fd0ce3fb640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fd0c808aae0 0x7fd0c812b990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:38.889 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.882+0000 7fd0ce3fb640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fd0c8135fb0 0x7fd0c812bed0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:38.889 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.882+0000 7fd0ce3fb640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fd0c8094830 0x7fd0c8094c70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:38.889 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.882+0000 7fd0ce3fb640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd0c8093c00 con 0x7fd0c808aae0 2024-09-21T09:20:38.889 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.882+0000 7fd0ce3fb640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd0c8092870 con 0x7fd0c8094830 2024-09-21T09:20:38.889 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.882+0000 7fd0ce3fb640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 v1 -- 0x7fd0c80958b0 con 0x7fd0c8135fb0 2024-09-21T09:20:38.889 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.883+0000 7fd0c77fe640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fd0c8135fb0 0x7fd0c812bed0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:38.889 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.883+0000 7fd0c77fe640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fd0c8135fb0 0x7fd0c812bed0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3302/0 says I am v2:172.21.15.78:43192/0 (socket says 172.21.15.78:43192) 2024-09-21T09:20:38.889 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.883+0000 7fd0c77fe640 1 -- 172.21.15.78:0/3073271320 learned_addr learned my addr 172.21.15.78:0/3073271320 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:20:38.889 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.883+0000 7fd0cc998640 1 --2- 172.21.15.78:0/3073271320 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fd0c8094830 0x7fd0c8094c70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:38.889 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.883+0000 7fd0c7fff640 1 --2- 172.21.15.78:0/3073271320 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fd0c808aae0 0x7fd0c812b990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:38.889 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.884+0000 7fd0cc998640 1 -- 172.21.15.78:0/3073271320 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fd0c8135fb0 msgr2=0x7fd0c812bed0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:38.889 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.884+0000 7fd0cc998640 1 --2- 172.21.15.78:0/3073271320 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fd0c8135fb0 0x7fd0c812bed0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.890 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.884+0000 7fd0cc998640 1 -- 172.21.15.78:0/3073271320 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fd0c808aae0 msgr2=0x7fd0c812b990 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:38.890 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.884+0000 7fd0cc998640 1 --2- 172.21.15.78:0/3073271320 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fd0c808aae0 0x7fd0c812b990 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.890 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.884+0000 7fd0cc998640 1 -- 172.21.15.78:0/3073271320 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd0c0009840 con 0x7fd0c8094830 2024-09-21T09:20:38.890 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.884+0000 7fd0c7fff640 1 --2- 172.21.15.78:0/3073271320 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fd0c808aae0 0x7fd0c812b990 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:20:38.890 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.884+0000 7fd0cc998640 1 --2- 172.21.15.78:0/3073271320 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fd0c8094830 0x7fd0c8094c70 secure :-1 s=READY pgs=24 cs=0 l=1 rev1=1 crypto rx=0x7fd0c000ee50 tx=0x7fd0c000c6a0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:20:38.890 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.885+0000 7fd0c4ff9640 1 -- 172.21.15.78:0/3073271320 <== mon.1 v2:172.21.15.78:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd0c0003970 con 0x7fd0c8094830 2024-09-21T09:20:38.890 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.885+0000 7fd0ce3fb640 1 -- 172.21.15.78:0/3073271320 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd0b8009490 con 0x7fd0c8094830 2024-09-21T09:20:38.890 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.884+0000 7efe2effd640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7efe28011be0 0x7efe280b52e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3300/0 says I am v2:172.21.15.78:35048/0 (socket says 172.21.15.78:35048) 2024-09-21T09:20:38.890 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.886+0000 7efe34be6640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7efe28012000 0x7efe2813ff20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:38.891 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.885+0000 7fd0ce3fb640 1 -- 172.21.15.78:0/3073271320 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd0c8096a50 con 0x7fd0c8094830 2024-09-21T09:20:38.891 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.886+0000 7efe34be6640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efe28141ae0 con 0x7efe28011be0 2024-09-21T09:20:38.891 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.886+0000 7efe34be6640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 v1 -- 0x7efe2800d150 con 0x7efe280112c0 2024-09-21T09:20:38.891 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.887+0000 7fd0ce3fb640 1 -- 172.21.15.78:0/3073271320 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd0c8092290 con 0x7fd0c8094830 2024-09-21T09:20:38.892 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.886+0000 7efe34be6640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 v1 -- 0x7efe28142db0 con 0x7efe28012000 2024-09-21T09:20:38.893 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.887+0000 7efe2f7fe640 1 -- 172.21.15.78:0/36108710 learned_addr learned my addr 172.21.15.78:0/36108710 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:20:38.893 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.887+0000 7efe2ffff640 1 --2- 172.21.15.78:0/36108710 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7efe28012000 0x7efe2813ff20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:38.893 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.888+0000 7efe2ffff640 1 -- 172.21.15.78:0/36108710 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7efe280112c0 msgr2=0x7efe280b4da0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:38.894 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.891+0000 7fd0c4ff9640 1 -- 172.21.15.78:0/3073271320 <== mon.1 v2:172.21.15.78:3301/0 2 ==== config(4 keys) v1 ==== 110+0+0 (secure 0 0 0) 0x7fd0c0003ba0 con 0x7fd0c8094830 2024-09-21T09:20:38.894 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.891+0000 7fd0c4ff9640 1 -- 172.21.15.78:0/3073271320 <== mon.1 v2:172.21.15.78:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd0c0004020 con 0x7fd0c8094830 2024-09-21T09:20:38.894 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.891+0000 7fd0c4ff9640 1 -- 172.21.15.78:0/3073271320 <== mon.1 v2:172.21.15.78:3301/0 4 ==== mgrmap(e 1) v1 ==== 668+0+0 (secure 0 0 0) 0x7fd0c0004b40 con 0x7fd0c8094830 2024-09-21T09:20:38.894 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.892+0000 7fd0c4ff9640 1 -- 172.21.15.78:0/3073271320 <== mon.1 v2:172.21.15.78:3301/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 725+0+0 (secure 0 0 0) 0x7fd0c0036070 con 0x7fd0c8094830 2024-09-21T09:20:38.894 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.892+0000 7fd0c4ff9640 1 -- 172.21.15.78:0/3073271320 <== mon.1 v2:172.21.15.78:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+73245 (secure 0 0 0) 0x7fd0c00054f0 con 0x7fd0c8094830 2024-09-21T09:20:38.896 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.894+0000 7f4645ffb640 1 -- 172.21.15.78:0/3561683452 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_command([{prefix=config set, name=mds_op_complaint_time}] v 0) v1 -- 0x7f46400050c0 con 0x7f466012a500 2024-09-21T09:20:38.897 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.895+0000 7efe2ffff640 1 --2- 172.21.15.78:0/36108710 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7efe280112c0 0x7efe280b4da0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.898 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.895+0000 7efe2ffff640 1 -- 172.21.15.78:0/36108710 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7efe28011be0 msgr2=0x7efe280b52e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:38.898 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.895+0000 7efe2ffff640 1 --2- 172.21.15.78:0/36108710 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7efe28011be0 0x7efe280b52e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.898 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.895+0000 7efe2ffff640 1 -- 172.21.15.78:0/36108710 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7efe1c009560 con 0x7efe28012000 2024-09-21T09:20:38.898 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.895+0000 7efe2effd640 1 --2- 172.21.15.78:0/36108710 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7efe28011be0 0x7efe280b52e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-09-21T09:20:38.898 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.895+0000 7efe2f7fe640 1 --2- 172.21.15.78:0/36108710 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7efe280112c0 0x7efe280b4da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-09-21T09:20:38.898 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.895+0000 7efe2ffff640 1 --2- 172.21.15.78:0/36108710 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7efe28012000 0x7efe2813ff20 secure :-1 s=READY pgs=18 cs=0 l=1 rev1=1 crypto rx=0x7efe1c031000 tx=0x7efe1c003b90 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:20:38.898 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.896+0000 7efe1bfff640 1 -- 172.21.15.78:0/36108710 <== mon.2 v2:172.21.15.78:3302/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efe1c004500 con 0x7efe28012000 2024-09-21T09:20:38.899 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.896+0000 7efe1bfff640 1 -- 172.21.15.78:0/36108710 <== mon.2 v2:172.21.15.78:3302/0 2 ==== config(4 keys) v1 ==== 110+0+0 (secure 0 0 0) 0x7efe1c03b460 con 0x7efe28012000 2024-09-21T09:20:38.899 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.896+0000 7efe1bfff640 1 -- 172.21.15.78:0/36108710 <== mon.2 v2:172.21.15.78:3302/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efe1c03b7f0 con 0x7efe28012000 2024-09-21T09:20:38.899 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.896+0000 7efe34be6640 1 -- 172.21.15.78:0/36108710 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7efe28017460 con 0x7efe28012000 2024-09-21T09:20:38.899 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.896+0000 7efe34be6640 1 -- 172.21.15.78:0/36108710 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7efe28142400 con 0x7efe28012000 2024-09-21T09:20:38.899 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.897+0000 7efe1bfff640 1 -- 172.21.15.78:0/36108710 <== mon.2 v2:172.21.15.78:3302/0 4 ==== mgrmap(e 1) v1 ==== 668+0+0 (secure 0 0 0) 0x7efe1c03b950 con 0x7efe28012000 2024-09-21T09:20:38.899 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.897+0000 7efe1bfff640 1 -- 172.21.15.78:0/36108710 <== mon.2 v2:172.21.15.78:3302/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 725+0+0 (secure 0 0 0) 0x7efe1c044d30 con 0x7efe28012000 2024-09-21T09:20:38.900 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.897+0000 7f143445a640 1 -- 172.21.15.78:0/2700596629 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_command([{prefix=config set, name=client_mount_timeout}] v 0) v1 -- 0x7f14140057d0 con 0x7f142c06ca60 2024-09-21T09:20:38.901 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.898+0000 7efe34be6640 1 -- 172.21.15.78:0/36108710 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7efe10005250 con 0x7efe28012000 2024-09-21T09:20:38.904 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.902+0000 7efe1bfff640 1 -- 172.21.15.78:0/36108710 <== mon.2 v2:172.21.15.78:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+73245 (secure 0 0 0) 0x7efe1c049060 con 0x7efe28012000 2024-09-21T09:20:38.906 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.904+0000 7f4647fff640 1 -- 172.21.15.78:0/3561683452 <== mon.1 v2:172.21.15.78:3301/0 7 ==== mon_command_ack([{prefix=config set, name=mds_op_complaint_time}]=0 v11)=0 v11) v1 ==== 121+0+0 (secure 0 0 0) 0x7f4658034020 con 0x7f466012a500 2024-09-21T09:20:38.916 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.914+0000 7f4645ffb640 1 -- 172.21.15.78:0/3561683452 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f466012a500 msgr2=0x7f466012de10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:38.916 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.914+0000 7f4645ffb640 1 --2- 172.21.15.78:0/3561683452 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f466012a500 0x7f466012de10 secure :-1 s=READY pgs=17 cs=0 l=1 rev1=1 crypto rx=0x7f466006ace0 tx=0x7f4658003f90 comp rx=0 tx=0).stop 2024-09-21T09:20:38.917 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.915+0000 7f4645ffb640 1 -- 172.21.15.78:0/3561683452 shutdown_connections 2024-09-21T09:20:38.917 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.915+0000 7f4645ffb640 1 --2- 172.21.15.78:0/3561683452 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f466012a500 0x7f466012de10 unknown :-1 s=CLOSED pgs=17 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.917 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.915+0000 7f4645ffb640 1 --2- 172.21.15.78:0/3561683452 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f4660070a30 0x7f4660095f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.917 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.915+0000 7f4645ffb640 1 --2- 172.21.15.78:0/3561683452 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f466006f420 0x7f46600959e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.917 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.915+0000 7f4645ffb640 1 -- 172.21.15.78:0/3561683452 >> 172.21.15.78:0/3561683452 conn(0x7f4660097190 msgr2=0x7f46601237f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:20:38.919 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.917+0000 7f4645ffb640 1 -- 172.21.15.78:0/3561683452 shutdown_connections 2024-09-21T09:20:38.919 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.917+0000 7f4645ffb640 1 -- 172.21.15.78:0/3561683452 wait complete. 2024-09-21T09:20:38.921 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.919+0000 7f063d7fa640 1 -- 172.21.15.78:0/4268584608 <== mon.1 v2:172.21.15.78:3301/0 7 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7f064006b610 con 0x7f063800d7c0 2024-09-21T09:20:38.923 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.921+0000 7f417affd640 1 -- 172.21.15.78:0/2752674021 <== mon.2 v2:172.21.15.78:3302/0 7 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7f4184005670 con 0x7f418c0898d0 2024-09-21T09:20:38.924 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.921+0000 7f4e617fa640 1 -- 172.21.15.78:0/529245718 <== mon.2 v2:172.21.15.78:3302/0 7 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7f4e5c02cab0 con 0x7f4e6408b410 2024-09-21T09:20:38.924 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.921+0000 7efe1bfff640 1 -- 172.21.15.78:0/36108710 <== mon.2 v2:172.21.15.78:3302/0 7 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7efe1c03bd40 con 0x7efe28012000 2024-09-21T09:20:38.924 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.922+0000 7f7271ffb640 1 -- 172.21.15.78:0/374465910 <== mon.2 v2:172.21.15.78:3302/0 7 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7f727400fe60 con 0x7f727c0898d0 2024-09-21T09:20:38.925 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.922+0000 7f7cc8ff9640 1 -- 172.21.15.78:0/2828725089 <== mon.2 v2:172.21.15.78:3302/0 7 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7f7ccc066180 con 0x7f7cc401a790 2024-09-21T09:20:38.927 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.923+0000 7f141effd640 1 -- 172.21.15.78:0/2700596629 <== mon.1 v2:172.21.15.78:3301/0 7 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7f1420004a50 con 0x7f142c06ca60 2024-09-21T09:20:38.927 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.923+0000 7f141effd640 1 -- 172.21.15.78:0/2700596629 <== mon.1 v2:172.21.15.78:3301/0 8 ==== mon_command_ack([{prefix=config set, name=client_mount_timeout}]=0 v12)=0 v12) v1 ==== 123+0+0 (secure 0 0 0) 0x7f1420004bc0 con 0x7f142c06ca60 2024-09-21T09:20:38.927 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.925+0000 7fd0c4ff9640 1 -- 172.21.15.78:0/3073271320 <== mon.1 v2:172.21.15.78:3301/0 7 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7fd0c0004df0 con 0x7fd0c8094830 2024-09-21T09:20:38.928 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.925+0000 7f143445a640 1 -- 172.21.15.78:0/2700596629 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f142c06ca60 msgr2=0x7f142c12d2b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:38.928 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.925+0000 7f143445a640 1 --2- 172.21.15.78:0/2700596629 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f142c06ca60 0x7f142c12d2b0 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7f1420004630 tx=0x7f142000da30 comp rx=0 tx=0).stop 2024-09-21T09:20:38.928 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.925+0000 7f143445a640 1 -- 172.21.15.78:0/2700596629 shutdown_connections 2024-09-21T09:20:38.928 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.925+0000 7f143445a640 1 --2- 172.21.15.78:0/2700596629 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f142c12ced0 0x7f142c12dd30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.928 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.925+0000 7f143445a640 1 --2- 172.21.15.78:0/2700596629 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f142c12c5b0 0x7f142c12d7f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.928 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.925+0000 7f143445a640 1 --2- 172.21.15.78:0/2700596629 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f142c06ca60 0x7f142c12d2b0 secure :-1 s=CLOSED pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7f1420004630 tx=0x7f142000da30 comp rx=0 tx=0).stop 2024-09-21T09:20:38.928 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.925+0000 7f143445a640 1 -- 172.21.15.78:0/2700596629 >> 172.21.15.78:0/2700596629 conn(0x7f142c09e9d0 msgr2=0x7f142c126120 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:20:38.930 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.926+0000 7f143445a640 1 -- 172.21.15.78:0/2700596629 shutdown_connections 2024-09-21T09:20:38.930 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.928+0000 7f143445a640 1 -- 172.21.15.78:0/2700596629 wait complete. 2024-09-21T09:20:38.939 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.931+0000 7f4e6acc5640 1 -- 172.21.15.78:0/529245718 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_command([{prefix=config set, name=osd_op_complaint_time}] v 0) v1 -- 0x7f4e440050c0 con 0x7f4e6408b410 2024-09-21T09:20:38.943 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.941+0000 7f7cd22ec640 1 -- 172.21.15.78:0/2828725089 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_command([{prefix=config set, name=rados_mon_op_timeout}] v 0) v1 -- 0x7f7cac0050c0 con 0x7f7cc401a790 2024-09-21T09:20:38.945 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.943+0000 7f4e617fa640 1 -- 172.21.15.78:0/529245718 <== mon.2 v2:172.21.15.78:3302/0 8 ==== mon_command_ack([{prefix=config set, name=osd_op_complaint_time}]=0 v13)=0 v13) v1 ==== 121+0+0 (secure 0 0 0) 0x7f4e5c044070 con 0x7f4e6408b410 2024-09-21T09:20:38.947 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.945+0000 7f06470d7640 1 -- 172.21.15.78:0/4268584608 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_command([{prefix=config set, name=mds_verify_scatter}] v 0) v1 -- 0x7f0618005140 con 0x7f063800d7c0 2024-09-21T09:20:38.948 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.945+0000 7f4e6acc5640 1 -- 172.21.15.78:0/529245718 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f4e6408b410 msgr2=0x7f4e6408e850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:38.948 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.945+0000 7f4e6acc5640 1 --2- 172.21.15.78:0/529245718 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f4e6408b410 0x7f4e6408e850 secure :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0x7f4e5c009e50 tx=0x7f4e5c02c720 comp rx=0 tx=0).stop 2024-09-21T09:20:38.948 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.945+0000 7f4e6acc5640 1 -- 172.21.15.78:0/529245718 shutdown_connections 2024-09-21T09:20:38.948 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.945+0000 7f4e6acc5640 1 --2- 172.21.15.78:0/529245718 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f4e64091f10 0x7f4e6408ed90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.948 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.945+0000 7f4e6acc5640 1 --2- 172.21.15.78:0/529245718 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f4e6408b410 0x7f4e6408e850 unknown :-1 s=CLOSED pgs=12 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.948 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.945+0000 7f4e6acc5640 1 --2- 172.21.15.78:0/529245718 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f4e64088ac0 0x7f4e6408e310 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.948 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.945+0000 7f4e6acc5640 1 -- 172.21.15.78:0/529245718 >> 172.21.15.78:0/529245718 conn(0x7f4e640986a0 msgr2=0x7f4e64088ea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:20:38.948 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.946+0000 7f4e6acc5640 1 -- 172.21.15.78:0/529245718 shutdown_connections 2024-09-21T09:20:38.948 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.946+0000 7f4e6acc5640 1 -- 172.21.15.78:0/529245718 wait complete. 2024-09-21T09:20:38.951 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'client', 'debug_client', '20'], check_status=True, hostname='smithi078') 2024-09-21T09:20:38.951 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'client', 'debug_ms', '1'], check_status=True, hostname='smithi078') 2024-09-21T09:20:38.951 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'client', 'rados_mon_op_timeout', '900'], check_status=True, hostname='smithi078') 2024-09-21T09:20:38.951 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'client', 'rados_osd_op_timeout', '900'], check_status=True, hostname='smithi078') 2024-09-21T09:20:38.951 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'mds', 'debug_mds', '20'], check_status=True, hostname='smithi078') 2024-09-21T09:20:38.951 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'mds', 'debug_mds_balancer', '20'], check_status=True, hostname='smithi078') 2024-09-21T09:20:38.952 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'mds', 'debug_ms', '2'], check_status=True, hostname='smithi078') 2024-09-21T09:20:38.952 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'mds', 'mds_debug_frag', 'True'], check_status=True, hostname='smithi078') 2024-09-21T09:20:38.952 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'mds', 'mds_debug_scatterstat', 'True'], check_status=True, hostname='smithi078') 2024-09-21T09:20:38.952 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'mds', 'mds_op_complaint_time', '180'], check_status=True, hostname='smithi078') 2024-09-21T09:20:38.952 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'mds', 'mds_verify_scatter', 'True'], check_status=True, hostname='smithi078') 2024-09-21T09:20:38.955 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.952+0000 7f7cc8ff9640 1 -- 172.21.15.78:0/2828725089 <== mon.2 v2:172.21.15.78:3302/0 8 ==== mon_command_ack([{prefix=config set, name=rados_mon_op_timeout}]=0 v14)=0 v14) v1 ==== 120+0+0 (secure 0 0 0) 0x7f7ccc092960 con 0x7f7cc401a790 2024-09-21T09:20:38.956 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.954+0000 7f7cb67fc640 1 -- 172.21.15.78:0/2828725089 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f7cc401a790 msgr2=0x7f7cc40ab5a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:38.957 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.954+0000 7f7cb67fc640 1 --2- 172.21.15.78:0/2828725089 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f7cc401a790 0x7f7cc40ab5a0 secure :-1 s=READY pgs=13 cs=0 l=1 rev1=1 crypto rx=0x7f7ccc069c80 tx=0x7f7ccc066d10 comp rx=0 tx=0).stop 2024-09-21T09:20:38.957 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.954+0000 7f7cb67fc640 1 -- 172.21.15.78:0/2828725089 shutdown_connections 2024-09-21T09:20:38.957 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.954+0000 7f7cb67fc640 1 --2- 172.21.15.78:0/2828725089 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f7cc40b2a90 0x7f7cc40ac020 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.957 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.954+0000 7f7cb67fc640 1 --2- 172.21.15.78:0/2828725089 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f7cc401b2f0 0x7f7cc40abae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.957 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.954+0000 7f7cb67fc640 1 --2- 172.21.15.78:0/2828725089 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f7cc401a790 0x7f7cc40ab5a0 unknown :-1 s=CLOSED pgs=13 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.957 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.954+0000 7f7cb67fc640 1 -- 172.21.15.78:0/2828725089 >> 172.21.15.78:0/2828725089 conn(0x7f7cc4006640 msgr2=0x7f7cc401c980 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:20:38.957 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.954+0000 7f7cb67fc640 1 -- 172.21.15.78:0/2828725089 shutdown_connections 2024-09-21T09:20:38.957 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.954+0000 7f7cb67fc640 1 -- 172.21.15.78:0/2828725089 wait complete. 2024-09-21T09:20:38.971 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.968+0000 7f4190cc9640 1 -- 172.21.15.78:0/2752674021 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_command([{prefix=config set, name=rados_osd_op_timeout}] v 0) v1 -- 0x7f4164005140 con 0x7f418c0898d0 2024-09-21T09:20:38.972 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.970+0000 7f063d7fa640 1 -- 172.21.15.78:0/4268584608 <== mon.1 v2:172.21.15.78:3301/0 8 ==== mon_command_ack([{prefix=config set, name=mds_verify_scatter}]=0 v15)=0 v15) v1 ==== 119+0+0 (secure 0 0 0) 0x7f06400672c0 con 0x7f063800d7c0 2024-09-21T09:20:38.973 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.971+0000 7f06470d7640 1 -- 172.21.15.78:0/4268584608 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f063800d7c0 msgr2=0x7f0638017770 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:38.974 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.971+0000 7f06470d7640 1 --2- 172.21.15.78:0/4268584608 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f063800d7c0 0x7f0638017770 secure :-1 s=READY pgs=21 cs=0 l=1 rev1=1 crypto rx=0x7f064004efc0 tx=0x7f0640073830 comp rx=0 tx=0).stop 2024-09-21T09:20:38.974 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.971+0000 7f06470d7640 1 -- 172.21.15.78:0/4268584608 shutdown_connections 2024-09-21T09:20:38.974 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.971+0000 7f06470d7640 1 --2- 172.21.15.78:0/4268584608 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f06381778f0 0x7f0638177d70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.974 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.971+0000 7f06470d7640 1 --2- 172.21.15.78:0/4268584608 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f063800d7c0 0x7f0638017770 secure :-1 s=CLOSED pgs=21 cs=0 l=1 rev1=1 crypto rx=0x7f064004efc0 tx=0x7f0640073830 comp rx=0 tx=0).stop 2024-09-21T09:20:38.974 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.971+0000 7f06470d7640 1 --2- 172.21.15.78:0/4268584608 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f063800cea0 0x7f0638017230 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.974 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.971+0000 7f06470d7640 1 -- 172.21.15.78:0/4268584608 >> 172.21.15.78:0/4268584608 conn(0x7f06380059e0 msgr2=0x7f063801de50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:20:38.974 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.972+0000 7f06470d7640 1 -- 172.21.15.78:0/4268584608 shutdown_connections 2024-09-21T09:20:38.974 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.972+0000 7f06470d7640 1 -- 172.21.15.78:0/4268584608 wait complete. 2024-09-21T09:20:38.982 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.980+0000 7f417affd640 1 -- 172.21.15.78:0/2752674021 <== mon.2 v2:172.21.15.78:3302/0 8 ==== mon_command_ack([{prefix=config set, name=rados_osd_op_timeout}]=0 v16)=0 v16) v1 ==== 120+0+0 (secure 0 0 0) 0x7f4184013030 con 0x7f418c0898d0 2024-09-21T09:20:38.984 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.981+0000 7f4190cc9640 1 -- 172.21.15.78:0/2752674021 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f418c0898d0 msgr2=0x7f418c12c170 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:38.984 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.981+0000 7f4190cc9640 1 --2- 172.21.15.78:0/2752674021 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f418c0898d0 0x7f418c12c170 secure :-1 s=READY pgs=15 cs=0 l=1 rev1=1 crypto rx=0x7f418401afc0 tx=0x7f418401b790 comp rx=0 tx=0).stop 2024-09-21T09:20:38.984 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.981+0000 7f4190cc9640 1 -- 172.21.15.78:0/2752674021 shutdown_connections 2024-09-21T09:20:38.985 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.981+0000 7f4190cc9640 1 --2- 172.21.15.78:0/2752674021 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f418c135010 0x7f418c08b0c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.985 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.981+0000 7f4190cc9640 1 --2- 172.21.15.78:0/2752674021 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f418c0898d0 0x7f418c12c170 unknown :-1 s=CLOSED pgs=15 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.985 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.981+0000 7f4190cc9640 1 --2- 172.21.15.78:0/2752674021 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f418c088fb0 0x7f418c12bc30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.985 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.981+0000 7f4190cc9640 1 -- 172.21.15.78:0/2752674021 >> 172.21.15.78:0/2752674021 conn(0x7f418c098570 msgr2=0x7f418c1346e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:20:38.985 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.982+0000 7f4190cc9640 1 -- 172.21.15.78:0/2752674021 shutdown_connections 2024-09-21T09:20:38.985 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.982+0000 7f4190cc9640 1 -- 172.21.15.78:0/2752674021 wait complete. 2024-09-21T09:20:38.985 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.983+0000 7f7283653640 1 -- 172.21.15.78:0/374465910 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_command([{prefix=config set, name=osd_op_complaint_time}] v 0) v1 -- 0x7f727c06d5c0 con 0x7f727c0898d0 2024-09-21T09:20:38.990 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'mds', 'osd_op_complaint_time', '180'], check_status=True, hostname='smithi078') 2024-09-21T09:20:38.990 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'mds', 'rados_mon_op_timeout', '900'], check_status=True, hostname='smithi078') 2024-09-21T09:20:38.991 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'mds', 'rados_osd_op_timeout', '900'], check_status=True, hostname='smithi078') 2024-09-21T09:20:38.992 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.990+0000 7f7271ffb640 1 -- 172.21.15.78:0/374465910 <== mon.2 v2:172.21.15.78:3302/0 8 ==== mon_command_ack([{prefix=config set, name=osd_op_complaint_time}]=0 v17)=0 v17) v1 ==== 121+0+0 (secure 0 0 0) 0x7f727401d020 con 0x7f727c0898d0 2024-09-21T09:20:38.995 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.992+0000 7f7283653640 1 -- 172.21.15.78:0/374465910 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f727c0898d0 msgr2=0x7f727c12b890 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:38.995 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.992+0000 7f7283653640 1 --2- 172.21.15.78:0/374465910 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f727c0898d0 0x7f727c12b890 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7f7274005fd0 tx=0x7f727400a620 comp rx=0 tx=0).stop 2024-09-21T09:20:38.995 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.992+0000 7f7283653640 1 -- 172.21.15.78:0/374465910 shutdown_connections 2024-09-21T09:20:38.995 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.992+0000 7f7283653640 1 --2- 172.21.15.78:0/374465910 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f727c135010 0x7f727c12bdd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.995 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.992+0000 7f7283653640 1 --2- 172.21.15.78:0/374465910 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f727c0898d0 0x7f727c12b890 unknown :-1 s=CLOSED pgs=16 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.995 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.992+0000 7f7283653640 1 --2- 172.21.15.78:0/374465910 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f727c088fb0 0x7f727c12b350 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:38.995 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.992+0000 7f7283653640 1 -- 172.21.15.78:0/374465910 >> 172.21.15.78:0/374465910 conn(0x7f727c098570 msgr2=0x7f727c127920 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:20:38.995 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.992+0000 7f7283653640 1 -- 172.21.15.78:0/374465910 shutdown_connections 2024-09-21T09:20:38.995 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:38.992+0000 7f7283653640 1 -- 172.21.15.78:0/374465910 wait complete. 2024-09-21T09:20:39.002 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'mon', 'mon_op_complaint_time', '120'], check_status=True, hostname='smithi078') 2024-09-21T09:20:39.017 INFO:tasks.ceph.mgr.y.smithi078.stderr:2024-09-21T09:20:39.015+0000 7ff872721040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:20:39.018 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.016+0000 7fd0ce3fb640 1 -- 172.21.15.78:0/3073271320 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_command([{prefix=config set, name=mon_warn_on_pool_no_app}] v 0) v1 -- 0x7fd0c808a1f0 con 0x7fd0c8094830 2024-09-21T09:20:39.020 INFO:tasks.ceph.mgr.x.smithi078.stderr:2024-09-21T09:20:39.018+0000 7fda56037040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:20:39.023 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.021+0000 7fd0c4ff9640 1 -- 172.21.15.78:0/3073271320 <== mon.1 v2:172.21.15.78:3301/0 8 ==== mon_command_ack([{prefix=config set, name=mon_warn_on_pool_no_app}]=0 v18)=0 v18) v1 ==== 125+0+0 (secure 0 0 0) 0x7fd0c00056b0 con 0x7fd0c8094830 2024-09-21T09:20:39.025 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.022+0000 7fd0ce3fb640 1 -- 172.21.15.78:0/3073271320 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fd0c8094830 msgr2=0x7fd0c8094c70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:39.025 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.022+0000 7fd0ce3fb640 1 --2- 172.21.15.78:0/3073271320 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fd0c8094830 0x7fd0c8094c70 secure :-1 s=READY pgs=24 cs=0 l=1 rev1=1 crypto rx=0x7fd0c000ee50 tx=0x7fd0c000c6a0 comp rx=0 tx=0).stop 2024-09-21T09:20:39.025 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.022+0000 7fd0ce3fb640 1 -- 172.21.15.78:0/3073271320 shutdown_connections 2024-09-21T09:20:39.025 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.022+0000 7fd0ce3fb640 1 --2- 172.21.15.78:0/3073271320 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fd0c8094830 0x7fd0c8094c70 unknown :-1 s=CLOSED pgs=24 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:39.025 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.022+0000 7fd0ce3fb640 1 --2- 172.21.15.78:0/3073271320 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fd0c8135fb0 0x7fd0c812bed0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:39.025 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.022+0000 7fd0ce3fb640 1 --2- 172.21.15.78:0/3073271320 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fd0c808aae0 0x7fd0c812b990 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:39.025 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.022+0000 7fd0ce3fb640 1 -- 172.21.15.78:0/3073271320 >> 172.21.15.78:0/3073271320 conn(0x7fd0c809a7e0 msgr2=0x7fd0c806cdd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:20:39.025 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.022+0000 7fd0ce3fb640 1 -- 172.21.15.78:0/3073271320 shutdown_connections 2024-09-21T09:20:39.025 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.022+0000 7fd0ce3fb640 1 -- 172.21.15.78:0/3073271320 wait complete. 2024-09-21T09:20:39.036 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.034+0000 7efe34be6640 1 -- 172.21.15.78:0/36108710 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_command([{prefix=config set, name=mon_op_complaint_time}] v 0) v1 -- 0x7efe100050c0 con 0x7efe28012000 2024-09-21T09:20:39.039 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.037+0000 7efe1bfff640 1 -- 172.21.15.78:0/36108710 <== mon.2 v2:172.21.15.78:3302/0 8 ==== mon_command_ack([{prefix=config set, name=mon_op_complaint_time}]=0 v19)=0 v19) v1 ==== 121+0+0 (secure 0 0 0) 0x7efe1c03f030 con 0x7efe28012000 2024-09-21T09:20:39.041 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.038+0000 7efe34be6640 1 -- 172.21.15.78:0/36108710 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7efe28012000 msgr2=0x7efe2813ff20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:39.041 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.038+0000 7efe34be6640 1 --2- 172.21.15.78:0/36108710 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7efe28012000 0x7efe2813ff20 secure :-1 s=READY pgs=18 cs=0 l=1 rev1=1 crypto rx=0x7efe1c031000 tx=0x7efe1c003b90 comp rx=0 tx=0).stop 2024-09-21T09:20:39.041 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.038+0000 7efe34be6640 1 -- 172.21.15.78:0/36108710 shutdown_connections 2024-09-21T09:20:39.041 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.038+0000 7efe34be6640 1 --2- 172.21.15.78:0/36108710 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7efe28012000 0x7efe2813ff20 unknown :-1 s=CLOSED pgs=18 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:39.041 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.038+0000 7efe34be6640 1 --2- 172.21.15.78:0/36108710 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7efe28011be0 0x7efe280b52e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:39.042 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.038+0000 7efe34be6640 1 --2- 172.21.15.78:0/36108710 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7efe280112c0 0x7efe280b4da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:39.042 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.038+0000 7efe34be6640 1 -- 172.21.15.78:0/36108710 >> 172.21.15.78:0/36108710 conn(0x7efe28005cf0 msgr2=0x7efe2800f390 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:20:39.042 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.039+0000 7efe34be6640 1 -- 172.21.15.78:0/36108710 shutdown_connections 2024-09-21T09:20:39.042 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.039+0000 7efe34be6640 1 -- 172.21.15.78:0/36108710 wait complete. 2024-09-21T09:20:39.051 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'mon', 'mon_warn_on_pool_no_app', 'False'], check_status=True, hostname='smithi078') 2024-09-21T09:20:39.051 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'osd', 'osd_op_complaint_time', '180'], check_status=True, hostname='smithi078') 2024-09-21T09:20:39.052 INFO:tasks.ceph:Setting crush tunables to default 2024-09-21T09:20:39.052 DEBUG:teuthology.orchestra.run.smithi078:> sudo ceph --cluster ceph osd crush tunables default 2024-09-21T09:20:39.136 INFO:tasks.ceph.mgr.y.smithi078.stderr:2024-09-21T09:20:39.133+0000 7ff872721040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-21T09:20:39.136 INFO:tasks.ceph.mgr.x.smithi078.stderr:2024-09-21T09:20:39.134+0000 7fda56037040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-21T09:20:39.193 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.190+0000 7f6ec160d640 1 -- 172.21.15.78:0/3255291180 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f6ebc06ee20 msgr2=0x7f6ebc12c0f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:39.193 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.190+0000 7f6ec160d640 1 --2- 172.21.15.78:0/3255291180 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f6ebc06ee20 0x7f6ebc12c0f0 secure :-1 s=READY pgs=25 cs=0 l=1 rev1=1 crypto rx=0x7f6ea4009830 tx=0x7f6ea402a360 comp rx=0 tx=0).stop 2024-09-21T09:20:39.193 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.190+0000 7f6ec160d640 1 -- 172.21.15.78:0/3255291180 shutdown_connections 2024-09-21T09:20:39.193 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.190+0000 7f6ec160d640 1 --2- 172.21.15.78:0/3255291180 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f6ebc12c630 0x7f6ebc12cac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:39.193 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.190+0000 7f6ec160d640 1 --2- 172.21.15.78:0/3255291180 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f6ebc06ee20 0x7f6ebc12c0f0 unknown :-1 s=CLOSED pgs=25 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:39.193 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.190+0000 7f6ec160d640 1 --2- 172.21.15.78:0/3255291180 >> v2:172.21.15.78:3300/0 conn(0x7f6ebc06e500 0x7f6ebc06e8e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:39.193 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.190+0000 7f6ec160d640 1 -- 172.21.15.78:0/3255291180 >> 172.21.15.78:0/3255291180 conn(0x7f6ebc097e40 msgr2=0x7f6ebc098250 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:20:39.194 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.190+0000 7f6ec160d640 1 -- 172.21.15.78:0/3255291180 shutdown_connections 2024-09-21T09:20:39.194 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.190+0000 7f6ec160d640 1 -- 172.21.15.78:0/3255291180 wait complete. 2024-09-21T09:20:39.196 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.192+0000 7f6ec160d640 1 Processor -- start 2024-09-21T09:20:39.196 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.192+0000 7f6ec160d640 1 -- start start 2024-09-21T09:20:39.196 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.192+0000 7f6ec160d640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f6ebc06e500 0x7f6ebc11aa20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:39.197 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.192+0000 7f6ec160d640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f6ebc06ee20 0x7f6ebc11af60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:39.197 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.192+0000 7f6ec160d640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f6ebc12c630 0x7f6ebc11b4a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:39.197 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.192+0000 7f6ec160d640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6ebc11cd70 con 0x7f6ebc06e500 2024-09-21T09:20:39.197 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.192+0000 7f6ec160d640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6ebc11cb60 con 0x7f6ebc06ee20 2024-09-21T09:20:39.197 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.192+0000 7f6ec160d640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 v1 -- 0x7f6ebc071040 con 0x7f6ebc12c630 2024-09-21T09:20:39.197 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.192+0000 7f6ebaffd640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f6ebc06e500 0x7f6ebc11aa20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:39.197 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.192+0000 7f6eb3fff640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f6ebc06ee20 0x7f6ebc11af60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:39.197 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.192+0000 7f6eb3fff640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f6ebc06ee20 0x7f6ebc11af60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3301/0 says I am v2:172.21.15.78:50312/0 (socket says 172.21.15.78:50312) 2024-09-21T09:20:39.197 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.192+0000 7f6ebaffd640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f6ebc06e500 0x7f6ebc11aa20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3300/0 says I am v2:172.21.15.78:35058/0 (socket says 172.21.15.78:35058) 2024-09-21T09:20:39.197 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.192+0000 7f6eb3fff640 1 -- 172.21.15.78:0/3941246769 learned_addr learned my addr 172.21.15.78:0/3941246769 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:20:39.197 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.192+0000 7f6ebb7fe640 1 --2- 172.21.15.78:0/3941246769 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f6ebc12c630 0x7f6ebc11b4a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:39.197 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.193+0000 7f6ebaffd640 1 -- 172.21.15.78:0/3941246769 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f6ebc12c630 msgr2=0x7f6ebc11b4a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:39.198 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.193+0000 7f6ebaffd640 1 --2- 172.21.15.78:0/3941246769 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f6ebc12c630 0x7f6ebc11b4a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:39.198 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.193+0000 7f6ebaffd640 1 -- 172.21.15.78:0/3941246769 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f6ebc06ee20 msgr2=0x7f6ebc11af60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:39.198 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.193+0000 7f6ebaffd640 1 --2- 172.21.15.78:0/3941246769 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f6ebc06ee20 0x7f6ebc11af60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:39.198 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.193+0000 7f6ebaffd640 1 -- 172.21.15.78:0/3941246769 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6eac009490 con 0x7f6ebc06e500 2024-09-21T09:20:39.198 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.193+0000 7f6eb3fff640 1 --2- 172.21.15.78:0/3941246769 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f6ebc06ee20 0x7f6ebc11af60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:20:39.198 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.193+0000 7f6ebaffd640 1 --2- 172.21.15.78:0/3941246769 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f6ebc06e500 0x7f6ebc11aa20 secure :-1 s=READY pgs=24 cs=0 l=1 rev1=1 crypto rx=0x7f6eac00ea20 tx=0x7f6eac00eee0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:20:39.198 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.193+0000 7f6eb37fe640 1 -- 172.21.15.78:0/3941246769 <== mon.0 v2:172.21.15.78:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6eac00ccf0 con 0x7f6ebc06e500 2024-09-21T09:20:39.198 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.193+0000 7f6eb37fe640 1 -- 172.21.15.78:0/3941246769 <== mon.0 v2:172.21.15.78:3300/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7f6eac00ce50 con 0x7f6ebc06e500 2024-09-21T09:20:39.198 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.193+0000 7f6ec160d640 1 -- 172.21.15.78:0/3941246769 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6ea4009490 con 0x7f6ebc06e500 2024-09-21T09:20:39.198 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.193+0000 7f6eb37fe640 1 -- 172.21.15.78:0/3941246769 <== mon.0 v2:172.21.15.78:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6eac004280 con 0x7f6ebc06e500 2024-09-21T09:20:39.198 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.193+0000 7f6ec160d640 1 -- 172.21.15.78:0/3941246769 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6ebc11c2d0 con 0x7f6ebc06e500 2024-09-21T09:20:39.198 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.193+0000 7f6eb37fe640 1 -- 172.21.15.78:0/3941246769 <== mon.0 v2:172.21.15.78:3300/0 4 ==== mgrmap(e 1) v1 ==== 668+0+0 (secure 0 0 0) 0x7f6eac0047c0 con 0x7f6ebc06e500 2024-09-21T09:20:39.199 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.193+0000 7f6eb37fe640 1 -- 172.21.15.78:0/3941246769 <== mon.0 v2:172.21.15.78:3300/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 725+0+0 (secure 0 0 0) 0x7f6eac014070 con 0x7f6ebc06e500 2024-09-21T09:20:39.199 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.194+0000 7f6ec160d640 1 -- 172.21.15.78:0/3941246769 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6ebc12c0f0 con 0x7f6ebc06e500 2024-09-21T09:20:39.199 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.196+0000 7f6eb37fe640 1 -- 172.21.15.78:0/3941246769 <== mon.0 v2:172.21.15.78:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+73245 (secure 0 0 0) 0x7f6eac0043e0 con 0x7f6ebc06e500 2024-09-21T09:20:39.261 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.259+0000 7f6ec160d640 1 -- 172.21.15.78:0/3941246769 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "osd crush tunables", "profile": "default"} v 0) v1 -- 0x7f6ebc073460 con 0x7f6ebc06e500 2024-09-21T09:20:39.267 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.264+0000 7f6eb37fe640 1 -- 172.21.15.78:0/3941246769 <== mon.0 v2:172.21.15.78:3300/0 7 ==== mon_command_ack([{"prefix": "osd crush tunables", "profile": "default"}]=0 adjusted tunables profile to default v2) v1 ==== 124+0+0 (secure 0 0 0) 0x7f6eac004ca0 con 0x7f6ebc06e500 2024-09-21T09:20:39.267 INFO:teuthology.orchestra.run.smithi078.stderr:adjusted tunables profile to default 2024-09-21T09:20:39.269 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.266+0000 7f6ec160d640 1 -- 172.21.15.78:0/3941246769 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f6ebc06e500 msgr2=0x7f6ebc11aa20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:39.269 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.266+0000 7f6ec160d640 1 --2- 172.21.15.78:0/3941246769 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f6ebc06e500 0x7f6ebc11aa20 secure :-1 s=READY pgs=24 cs=0 l=1 rev1=1 crypto rx=0x7f6eac00ea20 tx=0x7f6eac00eee0 comp rx=0 tx=0).stop 2024-09-21T09:20:39.269 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.266+0000 7f6ec160d640 1 -- 172.21.15.78:0/3941246769 shutdown_connections 2024-09-21T09:20:39.269 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.266+0000 7f6ec160d640 1 --2- 172.21.15.78:0/3941246769 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f6ebc12c630 0x7f6ebc11b4a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:39.269 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.266+0000 7f6ec160d640 1 --2- 172.21.15.78:0/3941246769 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f6ebc06ee20 0x7f6ebc11af60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:39.269 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.266+0000 7f6ec160d640 1 --2- 172.21.15.78:0/3941246769 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f6ebc06e500 0x7f6ebc11aa20 unknown :-1 s=CLOSED pgs=24 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:39.269 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.266+0000 7f6ec160d640 1 -- 172.21.15.78:0/3941246769 >> 172.21.15.78:0/3941246769 conn(0x7f6ebc097e40 msgr2=0x7f6ebc09aae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:20:39.269 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.266+0000 7f6ec160d640 1 -- 172.21.15.78:0/3941246769 shutdown_connections 2024-09-21T09:20:39.269 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.266+0000 7f6ec160d640 1 -- 172.21.15.78:0/3941246769 wait complete. 2024-09-21T09:20:39.279 INFO:tasks.ceph:check_enable_crimson: False 2024-09-21T09:20:39.279 INFO:tasks.ceph:Starting osd daemons in cluster ceph... 2024-09-21T09:20:39.279 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-21T09:20:39.279 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd if=/var/lib/ceph/osd/ceph-0/fsid of=/dev/stdout 2024-09-21T09:20:39.305 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-21T09:20:39.305 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd if=/var/lib/ceph/osd/ceph-1/fsid of=/dev/stdout 2024-09-21T09:20:39.369 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-21T09:20:39.369 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd if=/var/lib/ceph/osd/ceph-2/fsid of=/dev/stdout 2024-09-21T09:20:39.433 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-21T09:20:39.433 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd if=/var/lib/ceph/osd/ceph-3/fsid of=/dev/stdout 2024-09-21T09:20:39.453 INFO:tasks.ceph.mgr.x.smithi078.stderr:2024-09-21T09:20:39.450+0000 7fda56037040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-21T09:20:39.453 INFO:tasks.ceph.mgr.y.smithi078.stderr:2024-09-21T09:20:39.450+0000 7ff872721040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-21T09:20:39.477 DEBUG:teuthology.orchestra.run.smithi078:> sudo ceph --cluster ceph osd new 6b18d222-0705-4495-a01c-3e2ba2ab4cff 0 2024-09-21T09:20:39.504 INFO:tasks.ceph.mgr.x.smithi078.stderr:2024-09-21T09:20:39.502+0000 7fda56037040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-21T09:20:39.505 INFO:tasks.ceph.mgr.y.smithi078.stderr:2024-09-21T09:20:39.502+0000 7ff872721040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-21T09:20:39.605 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.602+0000 7f1bc9b65640 1 -- 172.21.15.78:0/1809683438 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f1bc41293f0 msgr2=0x7f1bc41341d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:39.605 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.602+0000 7f1bc9b65640 1 --2- 172.21.15.78:0/1809683438 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f1bc41293f0 0x7f1bc41341d0 secure :-1 s=READY pgs=26 cs=0 l=1 rev1=1 crypto rx=0x7f1bbc005fd0 tx=0x7f1bbc02a4b0 comp rx=0 tx=0).stop 2024-09-21T09:20:39.605 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.602+0000 7f1bc9b65640 1 -- 172.21.15.78:0/1809683438 shutdown_connections 2024-09-21T09:20:39.605 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.602+0000 7f1bc9b65640 1 --2- 172.21.15.78:0/1809683438 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f1bc41293f0 0x7f1bc41341d0 unknown :-1 s=CLOSED pgs=26 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:39.605 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.602+0000 7f1bc9b65640 1 --2- 172.21.15.78:0/1809683438 >> v2:172.21.15.78:3300/0 conn(0x7f1bc412c310 0x7f1bc412c6f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:39.605 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.602+0000 7f1bc9b65640 1 -- 172.21.15.78:0/1809683438 >> 172.21.15.78:0/1809683438 conn(0x7f1bc409c8a0 msgr2=0x7f1bc409ccb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:20:39.605 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.602+0000 7f1bc9b65640 1 -- 172.21.15.78:0/1809683438 shutdown_connections 2024-09-21T09:20:39.606 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.602+0000 7f1bc9b65640 1 -- 172.21.15.78:0/1809683438 wait complete. 2024-09-21T09:20:39.608 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.603+0000 7f1bc9b65640 1 Processor -- start 2024-09-21T09:20:39.608 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.603+0000 7f1bc9b65640 1 -- start start 2024-09-21T09:20:39.608 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.603+0000 7f1bc9b65640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f1bc41288b0 0x7f1bc41c2340 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:39.608 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.603+0000 7f1bc9b65640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f1bc41293f0 0x7f1bc41c2880 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:39.608 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.604+0000 7f1bc9b65640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f1bc412c310 0x7f1bc4130150 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:39.608 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.604+0000 7f1bc9b65640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1bc4132720 con 0x7f1bc412c310 2024-09-21T09:20:39.608 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.604+0000 7f1bc9b65640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1bc4132530 con 0x7f1bc41293f0 2024-09-21T09:20:39.608 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.604+0000 7f1bc9b65640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 v1 -- 0x7f1bc41329f0 con 0x7f1bc41288b0 2024-09-21T09:20:39.608 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.604+0000 7f1bc2ffd640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f1bc41293f0 0x7f1bc41c2880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:39.609 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.604+0000 7f1bc37fe640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f1bc41288b0 0x7f1bc41c2340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:39.609 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.604+0000 7f1bc2ffd640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f1bc41293f0 0x7f1bc41c2880 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3301/0 says I am v2:172.21.15.78:50322/0 (socket says 172.21.15.78:50322) 2024-09-21T09:20:39.609 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.604+0000 7f1bc2ffd640 1 -- 172.21.15.78:0/1995757257 learned_addr learned my addr 172.21.15.78:0/1995757257 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:20:39.609 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.604+0000 7f1bc3fff640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f1bc412c310 0x7f1bc4130150 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:39.609 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.604+0000 7f1bc2ffd640 1 -- 172.21.15.78:0/1995757257 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f1bc41288b0 msgr2=0x7f1bc41c2340 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:39.609 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.604+0000 7f1bc2ffd640 1 --2- 172.21.15.78:0/1995757257 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f1bc41288b0 0x7f1bc41c2340 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:39.609 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.604+0000 7f1bc2ffd640 1 -- 172.21.15.78:0/1995757257 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f1bc412c310 msgr2=0x7f1bc4130150 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:39.609 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.604+0000 7f1bc2ffd640 1 --2- 172.21.15.78:0/1995757257 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f1bc412c310 0x7f1bc4130150 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:39.609 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.604+0000 7f1bc2ffd640 1 -- 172.21.15.78:0/1995757257 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1bbc009560 con 0x7f1bc41293f0 2024-09-21T09:20:39.609 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.604+0000 7f1bc3fff640 1 --2- 172.21.15.78:0/1995757257 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f1bc412c310 0x7f1bc4130150 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:20:39.609 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.604+0000 7f1bc2ffd640 1 --2- 172.21.15.78:0/1995757257 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f1bc41293f0 0x7f1bc41c2880 secure :-1 s=READY pgs=27 cs=0 l=1 rev1=1 crypto rx=0x7f1bb800c450 tx=0x7f1bb800c910 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:20:39.609 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.604+0000 7f1babfff640 1 -- 172.21.15.78:0/1995757257 <== mon.1 v2:172.21.15.78:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1bb80052d0 con 0x7f1bc41293f0 2024-09-21T09:20:39.609 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.604+0000 7f1babfff640 1 -- 172.21.15.78:0/1995757257 <== mon.1 v2:172.21.15.78:3301/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7f1bb80035f0 con 0x7f1bc41293f0 2024-09-21T09:20:39.610 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.604+0000 7f1babfff640 1 -- 172.21.15.78:0/1995757257 <== mon.1 v2:172.21.15.78:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1bb8003a00 con 0x7f1bc41293f0 2024-09-21T09:20:39.610 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.604+0000 7f1bc9b65640 1 -- 172.21.15.78:0/1995757257 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1bc41265d0 con 0x7f1bc41293f0 2024-09-21T09:20:39.610 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.604+0000 7f1bc9b65640 1 -- 172.21.15.78:0/1995757257 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1bc41c2dc0 con 0x7f1bc41293f0 2024-09-21T09:20:39.610 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.605+0000 7f1babfff640 1 -- 172.21.15.78:0/1995757257 <== mon.1 v2:172.21.15.78:3301/0 4 ==== mgrmap(e 1) v1 ==== 668+0+0 (secure 0 0 0) 0x7f1bb8011590 con 0x7f1bc41293f0 2024-09-21T09:20:39.610 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.605+0000 7f1babfff640 1 -- 172.21.15.78:0/1995757257 <== mon.1 v2:172.21.15.78:3301/0 5 ==== osd_map(2..2 src has 1..2) v4 ==== 725+0+0 (secure 0 0 0) 0x7f1bb801c8c0 con 0x7f1bc41293f0 2024-09-21T09:20:39.610 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.605+0000 7f1bc9b65640 1 -- 172.21.15.78:0/1995757257 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1bc4129c10 con 0x7f1bc41293f0 2024-09-21T09:20:39.611 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.609+0000 7f1babfff640 1 -- 172.21.15.78:0/1995757257 <== mon.1 v2:172.21.15.78:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+73245 (secure 0 0 0) 0x7f1bc41265d0 con 0x7f1bc41293f0 2024-09-21T09:20:39.650 INFO:tasks.ceph.mgr.y.smithi078.stderr:2024-09-21T09:20:39.647+0000 7ff872721040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-21T09:20:39.650 INFO:tasks.ceph.mgr.x.smithi078.stderr:2024-09-21T09:20:39.648+0000 7fda56037040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-21T09:20:39.680 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.677+0000 7f1bc9b65640 1 -- 172.21.15.78:0/1995757257 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_command({"prefix": "osd new", "uuid": "6b18d222-0705-4495-a01c-3e2ba2ab4cff", "id": 0} v 0) v1 -- 0x7f1bc412c6f0 con 0x7f1bc41293f0 2024-09-21T09:20:39.685 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.683+0000 7f1babfff640 1 -- 172.21.15.78:0/1995757257 <== mon.1 v2:172.21.15.78:3301/0 7 ==== mon_command_ack([{"prefix": "osd new", "uuid": "6b18d222-0705-4495-a01c-3e2ba2ab4cff", "id": 0}]=0 v3) v1 ==== 112+0+2 (secure 0 0 0) 0x7f1bb8017070 con 0x7f1bc41293f0 2024-09-21T09:20:39.686 INFO:teuthology.orchestra.run.smithi078.stdout:0 2024-09-21T09:20:39.687 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.684+0000 7f1bc9b65640 1 -- 172.21.15.78:0/1995757257 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f1bc41293f0 msgr2=0x7f1bc41c2880 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:39.687 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.684+0000 7f1bc9b65640 1 --2- 172.21.15.78:0/1995757257 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f1bc41293f0 0x7f1bc41c2880 secure :-1 s=READY pgs=27 cs=0 l=1 rev1=1 crypto rx=0x7f1bb800c450 tx=0x7f1bb800c910 comp rx=0 tx=0).stop 2024-09-21T09:20:39.687 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.684+0000 7f1bc9b65640 1 -- 172.21.15.78:0/1995757257 shutdown_connections 2024-09-21T09:20:39.688 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.684+0000 7f1bc9b65640 1 --2- 172.21.15.78:0/1995757257 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f1bc412c310 0x7f1bc4130150 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:39.688 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.684+0000 7f1bc9b65640 1 --2- 172.21.15.78:0/1995757257 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f1bc41293f0 0x7f1bc41c2880 unknown :-1 s=CLOSED pgs=27 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:39.688 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.684+0000 7f1bc9b65640 1 --2- 172.21.15.78:0/1995757257 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f1bc41288b0 0x7f1bc41c2340 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:39.688 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.684+0000 7f1bc9b65640 1 -- 172.21.15.78:0/1995757257 >> 172.21.15.78:0/1995757257 conn(0x7f1bc409c8a0 msgr2=0x7f1bc41252a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:20:39.688 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.684+0000 7f1bc9b65640 1 -- 172.21.15.78:0/1995757257 shutdown_connections 2024-09-21T09:20:39.688 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.684+0000 7f1bc9b65640 1 -- 172.21.15.78:0/1995757257 wait complete. 2024-09-21T09:20:39.697 DEBUG:teuthology.orchestra.run.smithi078:> sudo ceph --cluster ceph osd new 2d15b0fa-66c2-4bda-b64b-d72cbbb16fb1 1 2024-09-21T09:20:39.801 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.794+0000 7fc6a3fed640 1 -- 172.21.15.78:0/1662117149 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fc69c096c30 msgr2=0x7fc69c097010 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:39.802 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.794+0000 7fc6a3fed640 1 --2- 172.21.15.78:0/1662117149 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fc69c096c30 0x7fc69c097010 secure :-1 s=READY pgs=28 cs=0 l=1 rev1=1 crypto rx=0x7fc698009900 tx=0x7fc69802a460 comp rx=0 tx=0).stop 2024-09-21T09:20:39.802 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.794+0000 7fc6a3fed640 1 -- 172.21.15.78:0/1662117149 shutdown_connections 2024-09-21T09:20:39.802 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.794+0000 7fc6a3fed640 1 --2- 172.21.15.78:0/1662117149 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fc69c071ab0 0x7fc69c071ee0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:39.802 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.794+0000 7fc6a3fed640 1 --2- 172.21.15.78:0/1662117149 >> v2:172.21.15.78:3300/0 conn(0x7fc69c097550 0x7fc69c071570 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:39.802 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.794+0000 7fc6a3fed640 1 --2- 172.21.15.78:0/1662117149 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fc69c096c30 0x7fc69c097010 unknown :-1 s=CLOSED pgs=28 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:39.803 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.794+0000 7fc6a3fed640 1 -- 172.21.15.78:0/1662117149 >> 172.21.15.78:0/1662117149 conn(0x7fc69c09ed50 msgr2=0x7fc69c09b220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:20:39.803 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.794+0000 7fc6a3fed640 1 -- 172.21.15.78:0/1662117149 shutdown_connections 2024-09-21T09:20:39.803 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.794+0000 7fc6a3fed640 1 -- 172.21.15.78:0/1662117149 wait complete. 2024-09-21T09:20:39.803 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.795+0000 7fc6a3fed640 1 Processor -- start 2024-09-21T09:20:39.803 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.795+0000 7fc6a3fed640 1 -- start start 2024-09-21T09:20:39.803 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.796+0000 7fc6a3fed640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fc69c071ab0 0x7fc69c132630 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:39.803 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.796+0000 7fc6a3fed640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fc69c096c30 0x7fc69c132b70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:39.804 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.796+0000 7fc6a3fed640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fc69c097550 0x7fc69c0942a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:39.804 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.796+0000 7fc6a3fed640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc69c08f140 con 0x7fc69c071ab0 2024-09-21T09:20:39.804 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.796+0000 7fc6a3fed640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc69c08f510 con 0x7fc69c097550 2024-09-21T09:20:39.804 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.796+0000 7fc6a3fed640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 v1 -- 0x7fc69c08e960 con 0x7fc69c096c30 2024-09-21T09:20:39.804 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.796+0000 7fc6a1d89640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fc69c071ab0 0x7fc69c132630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:39.804 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.796+0000 7fc6a1588640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fc69c096c30 0x7fc69c132b70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:39.805 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.796+0000 7fc6a1d89640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fc69c071ab0 0x7fc69c132630 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3300/0 says I am v2:172.21.15.78:35102/0 (socket says 172.21.15.78:35102) 2024-09-21T09:20:39.805 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.796+0000 7fc6a1d89640 1 -- 172.21.15.78:0/3729346490 learned_addr learned my addr 172.21.15.78:0/3729346490 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:20:39.805 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.796+0000 7fc6a258a640 1 --2- 172.21.15.78:0/3729346490 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fc69c097550 0x7fc69c0942a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:39.805 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.796+0000 7fc6a1d89640 1 -- 172.21.15.78:0/3729346490 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fc69c096c30 msgr2=0x7fc69c132b70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:39.805 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.796+0000 7fc6a1d89640 1 --2- 172.21.15.78:0/3729346490 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fc69c096c30 0x7fc69c132b70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:39.806 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.796+0000 7fc6a1d89640 1 -- 172.21.15.78:0/3729346490 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fc69c097550 msgr2=0x7fc69c0942a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:39.806 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.796+0000 7fc6a1d89640 1 --2- 172.21.15.78:0/3729346490 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fc69c097550 0x7fc69c0942a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:39.806 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.796+0000 7fc6a1d89640 1 -- 172.21.15.78:0/3729346490 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc68c009490 con 0x7fc69c071ab0 2024-09-21T09:20:39.806 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.796+0000 7fc6a1588640 1 --2- 172.21.15.78:0/3729346490 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fc69c096c30 0x7fc69c132b70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:20:39.806 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.796+0000 7fc6a258a640 1 --2- 172.21.15.78:0/3729346490 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fc69c097550 0x7fc69c0942a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:20:39.806 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.796+0000 7fc6a1d89640 1 --2- 172.21.15.78:0/3729346490 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fc69c071ab0 0x7fc69c132630 secure :-1 s=READY pgs=26 cs=0 l=1 rev1=1 crypto rx=0x7fc698006010 tx=0x7fc6980038a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:20:39.807 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.796+0000 7fc6927fc640 1 -- 172.21.15.78:0/3729346490 <== mon.0 v2:172.21.15.78:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc698003ea0 con 0x7fc69c071ab0 2024-09-21T09:20:39.807 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.797+0000 7fc6927fc640 1 -- 172.21.15.78:0/3729346490 <== mon.0 v2:172.21.15.78:3300/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7fc69803c470 con 0x7fc69c071ab0 2024-09-21T09:20:39.807 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.797+0000 7fc6a3fed640 1 -- 172.21.15.78:0/3729346490 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc698009560 con 0x7fc69c071ab0 2024-09-21T09:20:39.807 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.797+0000 7fc6927fc640 1 -- 172.21.15.78:0/3729346490 <== mon.0 v2:172.21.15.78:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc69803c880 con 0x7fc69c071ab0 2024-09-21T09:20:39.807 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.797+0000 7fc6a3fed640 1 -- 172.21.15.78:0/3729346490 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc69c08dfb0 con 0x7fc69c071ab0 2024-09-21T09:20:39.807 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.797+0000 7fc6927fc640 1 -- 172.21.15.78:0/3729346490 <== mon.0 v2:172.21.15.78:3300/0 4 ==== mgrmap(e 1) v1 ==== 668+0+0 (secure 0 0 0) 0x7fc69803cce0 con 0x7fc69c071ab0 2024-09-21T09:20:39.807 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.797+0000 7fc6927fc640 1 -- 172.21.15.78:0/3729346490 <== mon.0 v2:172.21.15.78:3300/0 5 ==== osd_map(3..3 src has 1..3) v4 ==== 840+0+0 (secure 0 0 0) 0x7fc6980317b0 con 0x7fc69c071ab0 2024-09-21T09:20:39.808 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.798+0000 7fc6a3fed640 1 -- 172.21.15.78:0/3729346490 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc69c08e260 con 0x7fc69c071ab0 2024-09-21T09:20:39.808 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.800+0000 7fc6927fc640 1 -- 172.21.15.78:0/3729346490 <== mon.0 v2:172.21.15.78:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+73245 (secure 0 0 0) 0x7fc698064e30 con 0x7fc69c071ab0 2024-09-21T09:20:39.885 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.882+0000 7fc6a3fed640 1 -- 172.21.15.78:0/3729346490 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "osd new", "uuid": "2d15b0fa-66c2-4bda-b64b-d72cbbb16fb1", "id": 1} v 0) v1 -- 0x7fc69c097010 con 0x7fc69c071ab0 2024-09-21T09:20:39.888 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.886+0000 7fc6927fc640 1 -- 172.21.15.78:0/3729346490 <== mon.0 v2:172.21.15.78:3300/0 7 ==== mon_command_ack([{"prefix": "osd new", "uuid": "2d15b0fa-66c2-4bda-b64b-d72cbbb16fb1", "id": 1}]=0 v4) v1 ==== 112+0+2 (secure 0 0 0) 0x7fc698053020 con 0x7fc69c071ab0 2024-09-21T09:20:39.888 INFO:teuthology.orchestra.run.smithi078.stdout:1 2024-09-21T09:20:39.889 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.887+0000 7fc6a3fed640 1 -- 172.21.15.78:0/3729346490 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fc69c071ab0 msgr2=0x7fc69c132630 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:39.889 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.887+0000 7fc6a3fed640 1 --2- 172.21.15.78:0/3729346490 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fc69c071ab0 0x7fc69c132630 secure :-1 s=READY pgs=26 cs=0 l=1 rev1=1 crypto rx=0x7fc698006010 tx=0x7fc6980038a0 comp rx=0 tx=0).stop 2024-09-21T09:20:39.889 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.887+0000 7fc6a3fed640 1 -- 172.21.15.78:0/3729346490 shutdown_connections 2024-09-21T09:20:39.889 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.887+0000 7fc6a3fed640 1 --2- 172.21.15.78:0/3729346490 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fc69c097550 0x7fc69c0942a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:39.889 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.887+0000 7fc6a3fed640 1 --2- 172.21.15.78:0/3729346490 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fc69c096c30 0x7fc69c132b70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:39.890 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.887+0000 7fc6a3fed640 1 --2- 172.21.15.78:0/3729346490 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fc69c071ab0 0x7fc69c132630 unknown :-1 s=CLOSED pgs=26 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:39.890 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.887+0000 7fc6a3fed640 1 -- 172.21.15.78:0/3729346490 >> 172.21.15.78:0/3729346490 conn(0x7fc69c09ed50 msgr2=0x7fc69c099f20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:20:39.890 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.887+0000 7fc6a3fed640 1 -- 172.21.15.78:0/3729346490 shutdown_connections 2024-09-21T09:20:39.890 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:39.887+0000 7fc6a3fed640 1 -- 172.21.15.78:0/3729346490 wait complete. 2024-09-21T09:20:39.900 DEBUG:teuthology.orchestra.run.smithi078:> sudo ceph --cluster ceph osd new 0bf75d8f-017d-488d-8a2e-c3fa3b2a29c2 2 2024-09-21T09:20:39.985 INFO:tasks.ceph.mgr.x.smithi078.stderr:2024-09-21T09:20:39.983+0000 7fda56037040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-21T09:20:39.993 INFO:tasks.ceph.mgr.y.smithi078.stderr:2024-09-21T09:20:39.991+0000 7ff872721040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-21T09:20:40.028 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.025+0000 7fae98165640 1 -- 172.21.15.78:0/3129870151 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fae9006f150 msgr2=0x7fae9006f530 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:40.028 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.025+0000 7fae98165640 1 --2- 172.21.15.78:0/3129870151 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fae9006f150 0x7fae9006f530 secure :-1 s=READY pgs=28 cs=0 l=1 rev1=1 crypto rx=0x7fae8c005fd0 tx=0x7fae8c02a4b0 comp rx=0 tx=0).stop 2024-09-21T09:20:40.028 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.025+0000 7fae98165640 1 -- 172.21.15.78:0/3129870151 shutdown_connections 2024-09-21T09:20:40.028 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.025+0000 7fae98165640 1 --2- 172.21.15.78:0/3129870151 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fae9012a4e0 0x7fae9012a900 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:40.028 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.025+0000 7fae98165640 1 --2- 172.21.15.78:0/3129870151 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fae9006f150 0x7fae9006f530 unknown :-1 s=CLOSED pgs=28 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:40.028 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.025+0000 7fae98165640 1 -- 172.21.15.78:0/3129870151 >> 172.21.15.78:0/3129870151 conn(0x7fae900972a0 msgr2=0x7fae900976b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:20:40.028 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.025+0000 7fae98165640 1 -- 172.21.15.78:0/3129870151 shutdown_connections 2024-09-21T09:20:40.028 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.025+0000 7fae98165640 1 -- 172.21.15.78:0/3129870151 wait complete. 2024-09-21T09:20:40.031 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.026+0000 7fae98165640 1 Processor -- start 2024-09-21T09:20:40.031 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.026+0000 7fae98165640 1 -- start start 2024-09-21T09:20:40.031 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.027+0000 7fae98165640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fae9006f150 0x7fae90125110 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:40.031 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.027+0000 7fae98165640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fae9006fa70 0x7fae90125650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:40.031 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.027+0000 7fae98165640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fae9012a4e0 0x7fae90125b90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:40.031 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.027+0000 7fae98165640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fae90131180 con 0x7fae9006f150 2024-09-21T09:20:40.031 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.027+0000 7fae98165640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fae90071bc0 con 0x7fae9006fa70 2024-09-21T09:20:40.031 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.027+0000 7fae98165640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 v1 -- 0x7fae90126b00 con 0x7fae9012a4e0 2024-09-21T09:20:40.032 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.027+0000 7fae95700640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fae9006fa70 0x7fae90125650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:40.032 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.027+0000 7fae95f01640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fae9006f150 0x7fae90125110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:40.032 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.027+0000 7fae95700640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fae9006fa70 0x7fae90125650 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3301/0 says I am v2:172.21.15.78:50356/0 (socket says 172.21.15.78:50356) 2024-09-21T09:20:40.032 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.027+0000 7fae95700640 1 -- 172.21.15.78:0/2832357163 learned_addr learned my addr 172.21.15.78:0/2832357163 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:20:40.032 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.027+0000 7fae96702640 1 --2- 172.21.15.78:0/2832357163 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fae9012a4e0 0x7fae90125b90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:40.032 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.027+0000 7fae95700640 1 -- 172.21.15.78:0/2832357163 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fae9012a4e0 msgr2=0x7fae90125b90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:40.032 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.027+0000 7fae95700640 1 --2- 172.21.15.78:0/2832357163 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fae9012a4e0 0x7fae90125b90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:40.032 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.027+0000 7fae95700640 1 -- 172.21.15.78:0/2832357163 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fae9006f150 msgr2=0x7fae90125110 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:40.032 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.027+0000 7fae95700640 1 --2- 172.21.15.78:0/2832357163 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fae9006f150 0x7fae90125110 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:40.032 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.027+0000 7fae95700640 1 -- 172.21.15.78:0/2832357163 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fae88009840 con 0x7fae9006fa70 2024-09-21T09:20:40.032 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.027+0000 7fae95f01640 1 --2- 172.21.15.78:0/2832357163 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fae9006f150 0x7fae90125110 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:20:40.032 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.027+0000 7fae96702640 1 --2- 172.21.15.78:0/2832357163 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fae9012a4e0 0x7fae90125b90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:20:40.032 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.027+0000 7fae95700640 1 --2- 172.21.15.78:0/2832357163 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fae9006fa70 0x7fae90125650 secure :-1 s=READY pgs=29 cs=0 l=1 rev1=1 crypto rx=0x7fae8000c430 tx=0x7fae8000c8f0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:20:40.032 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.027+0000 7fae867fc640 1 -- 172.21.15.78:0/2832357163 <== mon.1 v2:172.21.15.78:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fae800035f0 con 0x7fae9006fa70 2024-09-21T09:20:40.033 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.028+0000 7fae867fc640 1 -- 172.21.15.78:0/2832357163 <== mon.1 v2:172.21.15.78:3301/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7fae80003750 con 0x7fae9006fa70 2024-09-21T09:20:40.033 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.028+0000 7fae867fc640 1 -- 172.21.15.78:0/2832357163 <== mon.1 v2:172.21.15.78:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fae80003ba0 con 0x7fae9006fa70 2024-09-21T09:20:40.033 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.028+0000 7fae98165640 1 -- 172.21.15.78:0/2832357163 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fae8c009560 con 0x7fae9006fa70 2024-09-21T09:20:40.033 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.028+0000 7fae98165640 1 -- 172.21.15.78:0/2832357163 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fae90131c50 con 0x7fae9006fa70 2024-09-21T09:20:40.033 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.028+0000 7fae867fc640 1 -- 172.21.15.78:0/2832357163 <== mon.1 v2:172.21.15.78:3301/0 4 ==== mgrmap(e 1) v1 ==== 668+0+0 (secure 0 0 0) 0x7fae8001c440 con 0x7fae9006fa70 2024-09-21T09:20:40.033 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.028+0000 7fae867fc640 1 -- 172.21.15.78:0/2832357163 <== mon.1 v2:172.21.15.78:3301/0 5 ==== osd_map(4..4 src has 1..4) v4 ==== 955+0+0 (secure 0 0 0) 0x7fae80012820 con 0x7fae9006fa70 2024-09-21T09:20:40.033 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.029+0000 7fae98165640 1 -- 172.21.15.78:0/2832357163 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fae9012bcc0 con 0x7fae9006fa70 2024-09-21T09:20:40.033 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.031+0000 7fae867fc640 1 -- 172.21.15.78:0/2832357163 <== mon.1 v2:172.21.15.78:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+73245 (secure 0 0 0) 0x7fae80017070 con 0x7fae9006fa70 2024-09-21T09:20:40.039 INFO:tasks.ceph.mgr.x.smithi078.stderr:2024-09-21T09:20:40.037+0000 7fda56037040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-21T09:20:40.047 INFO:tasks.ceph.mgr.y.smithi078.stderr:2024-09-21T09:20:40.045+0000 7ff872721040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-21T09:20:40.102 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.100+0000 7fae98165640 1 -- 172.21.15.78:0/2832357163 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_command({"prefix": "osd new", "uuid": "0bf75d8f-017d-488d-8a2e-c3fa3b2a29c2", "id": 2} v 0) v1 -- 0x7fae9006f530 con 0x7fae9006fa70 2024-09-21T09:20:40.105 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.103+0000 7fae867fc640 1 -- 172.21.15.78:0/2832357163 <== mon.1 v2:172.21.15.78:3301/0 7 ==== mon_command_ack([{"prefix": "osd new", "uuid": "0bf75d8f-017d-488d-8a2e-c3fa3b2a29c2", "id": 2}]=0 v5) v1 ==== 112+0+2 (secure 0 0 0) 0x7fae80012510 con 0x7fae9006fa70 2024-09-21T09:20:40.106 INFO:teuthology.orchestra.run.smithi078.stdout:2 2024-09-21T09:20:40.107 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.105+0000 7fae98165640 1 -- 172.21.15.78:0/2832357163 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fae9006fa70 msgr2=0x7fae90125650 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:40.107 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.105+0000 7fae98165640 1 --2- 172.21.15.78:0/2832357163 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fae9006fa70 0x7fae90125650 secure :-1 s=READY pgs=29 cs=0 l=1 rev1=1 crypto rx=0x7fae8000c430 tx=0x7fae8000c8f0 comp rx=0 tx=0).stop 2024-09-21T09:20:40.107 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.105+0000 7fae98165640 1 -- 172.21.15.78:0/2832357163 shutdown_connections 2024-09-21T09:20:40.107 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.105+0000 7fae98165640 1 --2- 172.21.15.78:0/2832357163 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fae9012a4e0 0x7fae90125b90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:40.107 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.105+0000 7fae98165640 1 --2- 172.21.15.78:0/2832357163 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fae9006fa70 0x7fae90125650 unknown :-1 s=CLOSED pgs=29 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:40.107 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.105+0000 7fae98165640 1 --2- 172.21.15.78:0/2832357163 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fae9006f150 0x7fae90125110 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:40.108 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.105+0000 7fae98165640 1 -- 172.21.15.78:0/2832357163 >> 172.21.15.78:0/2832357163 conn(0x7fae900972a0 msgr2=0x7fae900920b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:20:40.108 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.105+0000 7fae98165640 1 -- 172.21.15.78:0/2832357163 shutdown_connections 2024-09-21T09:20:40.108 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.105+0000 7fae98165640 1 -- 172.21.15.78:0/2832357163 wait complete. 2024-09-21T09:20:40.121 DEBUG:teuthology.orchestra.run.smithi078:> sudo ceph --cluster ceph osd new 5e3788c8-b89a-4614-b3ec-7ee1250ddbe0 3 2024-09-21T09:20:40.128 INFO:tasks.ceph.mgr.x.smithi078.stderr:2024-09-21T09:20:40.126+0000 7fda56037040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-21T09:20:40.139 INFO:tasks.ceph.mgr.y.smithi078.stderr:2024-09-21T09:20:40.137+0000 7ff872721040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-21T09:20:40.221 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.219+0000 7f80cdcd7640 1 -- 172.21.15.78:0/2202260328 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f80c81293f0 msgr2=0x7f80c81341d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:40.221 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.219+0000 7f80cdcd7640 1 --2- 172.21.15.78:0/2202260328 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f80c81293f0 0x7f80c81341d0 secure :-1 s=READY pgs=30 cs=0 l=1 rev1=1 crypto rx=0x7f80c0005fd0 tx=0x7f80c002a4b0 comp rx=0 tx=0).stop 2024-09-21T09:20:40.222 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.219+0000 7f80cdcd7640 1 -- 172.21.15.78:0/2202260328 shutdown_connections 2024-09-21T09:20:40.222 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.219+0000 7f80cdcd7640 1 --2- 172.21.15.78:0/2202260328 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f80c81293f0 0x7f80c81341d0 unknown :-1 s=CLOSED pgs=30 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:40.222 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.219+0000 7f80cdcd7640 1 --2- 172.21.15.78:0/2202260328 >> v2:172.21.15.78:3300/0 conn(0x7f80c812c310 0x7f80c812c6f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:40.222 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.219+0000 7f80cdcd7640 1 -- 172.21.15.78:0/2202260328 >> 172.21.15.78:0/2202260328 conn(0x7f80c809c8a0 msgr2=0x7f80c809ccb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:20:40.222 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.219+0000 7f80cdcd7640 1 -- 172.21.15.78:0/2202260328 shutdown_connections 2024-09-21T09:20:40.222 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.219+0000 7f80cdcd7640 1 -- 172.21.15.78:0/2202260328 wait complete. 2024-09-21T09:20:40.222 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.220+0000 7f80cdcd7640 1 Processor -- start 2024-09-21T09:20:40.223 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.220+0000 7f80cdcd7640 1 -- start start 2024-09-21T09:20:40.224 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.221+0000 7f80cdcd7640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f80c81288b0 0x7f80c8130430 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:40.224 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.221+0000 7f80cdcd7640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f80c81293f0 0x7f80c8130970 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:40.224 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.221+0000 7f80cdcd7640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f80c812c310 0x7f80c8130eb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:40.224 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.221+0000 7f80cdcd7640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f80c81337f0 con 0x7f80c812c310 2024-09-21T09:20:40.224 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.221+0000 7f80cdcd7640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f80c8131af0 con 0x7f80c81293f0 2024-09-21T09:20:40.224 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.221+0000 7f80cdcd7640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 v1 -- 0x7f80c8126590 con 0x7f80c81288b0 2024-09-21T09:20:40.225 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.221+0000 7f80c6ffd640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f80c81293f0 0x7f80c8130970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:40.225 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.221+0000 7f80c7fff640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f80c812c310 0x7f80c8130eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:40.225 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.221+0000 7f80c6ffd640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f80c81293f0 0x7f80c8130970 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3301/0 says I am v2:172.21.15.78:50374/0 (socket says 172.21.15.78:50374) 2024-09-21T09:20:40.225 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.221+0000 7f80c6ffd640 1 -- 172.21.15.78:0/2924227888 learned_addr learned my addr 172.21.15.78:0/2924227888 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:20:40.225 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.221+0000 7f80c77fe640 1 --2- 172.21.15.78:0/2924227888 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f80c81288b0 0x7f80c8130430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:40.225 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.221+0000 7f80c7fff640 1 -- 172.21.15.78:0/2924227888 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f80c81288b0 msgr2=0x7f80c8130430 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:40.225 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.221+0000 7f80c7fff640 1 --2- 172.21.15.78:0/2924227888 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f80c81288b0 0x7f80c8130430 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:40.225 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.221+0000 7f80c7fff640 1 -- 172.21.15.78:0/2924227888 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f80c81293f0 msgr2=0x7f80c8130970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:40.225 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.221+0000 7f80c7fff640 1 --2- 172.21.15.78:0/2924227888 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f80c81293f0 0x7f80c8130970 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:40.225 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.221+0000 7f80c7fff640 1 -- 172.21.15.78:0/2924227888 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f80c0009560 con 0x7f80c812c310 2024-09-21T09:20:40.225 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.221+0000 7f80c6ffd640 1 --2- 172.21.15.78:0/2924227888 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f80c81293f0 0x7f80c8130970 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:20:40.225 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.222+0000 7f80c7fff640 1 --2- 172.21.15.78:0/2924227888 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f80c812c310 0x7f80c8130eb0 secure :-1 s=READY pgs=30 cs=0 l=1 rev1=1 crypto rx=0x7f80c0031000 tx=0x7f80c00038a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:20:40.225 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.222+0000 7f80affff640 1 -- 172.21.15.78:0/2924227888 <== mon.0 v2:172.21.15.78:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f80c0003ea0 con 0x7f80c812c310 2024-09-21T09:20:40.225 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.222+0000 7f80affff640 1 -- 172.21.15.78:0/2924227888 <== mon.0 v2:172.21.15.78:3300/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7f80c003c470 con 0x7f80c812c310 2024-09-21T09:20:40.226 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.222+0000 7f80cdcd7640 1 -- 172.21.15.78:0/2924227888 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f80c8132320 con 0x7f80c812c310 2024-09-21T09:20:40.226 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.222+0000 7f80affff640 1 -- 172.21.15.78:0/2924227888 <== mon.0 v2:172.21.15.78:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f80c003c880 con 0x7f80c812c310 2024-09-21T09:20:40.226 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.222+0000 7f80cdcd7640 1 -- 172.21.15.78:0/2924227888 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f80c8132f90 con 0x7f80c812c310 2024-09-21T09:20:40.226 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.222+0000 7f80affff640 1 -- 172.21.15.78:0/2924227888 <== mon.0 v2:172.21.15.78:3300/0 4 ==== mgrmap(e 1) v1 ==== 668+0+0 (secure 0 0 0) 0x7f80c003cce0 con 0x7f80c812c310 2024-09-21T09:20:40.226 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.222+0000 7f80affff640 1 -- 172.21.15.78:0/2924227888 <== mon.0 v2:172.21.15.78:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1070+0+0 (secure 0 0 0) 0x7f80c0045cc0 con 0x7f80c812c310 2024-09-21T09:20:40.226 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.223+0000 7f80cdcd7640 1 -- 172.21.15.78:0/2924227888 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f80c8127530 con 0x7f80c812c310 2024-09-21T09:20:40.227 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.225+0000 7f80affff640 1 -- 172.21.15.78:0/2924227888 <== mon.0 v2:172.21.15.78:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+73245 (secure 0 0 0) 0x7f80c004a060 con 0x7f80c812c310 2024-09-21T09:20:40.227 INFO:tasks.ceph.mgr.x.smithi078.stderr:2024-09-21T09:20:40.225+0000 7fda56037040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:20:40.247 INFO:tasks.ceph.mgr.y.smithi078.stderr:2024-09-21T09:20:40.245+0000 7ff872721040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:20:40.276 INFO:tasks.ceph.mgr.x.smithi078.stderr:2024-09-21T09:20:40.274+0000 7fda56037040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-21T09:20:40.309 INFO:tasks.ceph.mgr.y.smithi078.stderr:2024-09-21T09:20:40.307+0000 7ff872721040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-21T09:20:40.331 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.329+0000 7f80cdcd7640 1 -- 172.21.15.78:0/2924227888 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "osd new", "uuid": "5e3788c8-b89a-4614-b3ec-7ee1250ddbe0", "id": 3} v 0) v1 -- 0x7f80c812c6f0 con 0x7f80c812c310 2024-09-21T09:20:40.334 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.332+0000 7f80affff640 1 -- 172.21.15.78:0/2924227888 <== mon.0 v2:172.21.15.78:3300/0 7 ==== mon_command_ack([{"prefix": "osd new", "uuid": "5e3788c8-b89a-4614-b3ec-7ee1250ddbe0", "id": 3}]=0 v6) v1 ==== 112+0+2 (secure 0 0 0) 0x7f80c0040070 con 0x7f80c812c310 2024-09-21T09:20:40.334 INFO:teuthology.orchestra.run.smithi078.stdout:3 2024-09-21T09:20:40.335 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.333+0000 7f80cdcd7640 1 -- 172.21.15.78:0/2924227888 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f80c812c310 msgr2=0x7f80c8130eb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:40.335 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.333+0000 7f80cdcd7640 1 --2- 172.21.15.78:0/2924227888 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f80c812c310 0x7f80c8130eb0 secure :-1 s=READY pgs=30 cs=0 l=1 rev1=1 crypto rx=0x7f80c0031000 tx=0x7f80c00038a0 comp rx=0 tx=0).stop 2024-09-21T09:20:40.335 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.333+0000 7f80cdcd7640 1 -- 172.21.15.78:0/2924227888 shutdown_connections 2024-09-21T09:20:40.335 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.333+0000 7f80cdcd7640 1 --2- 172.21.15.78:0/2924227888 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f80c812c310 0x7f80c8130eb0 unknown :-1 s=CLOSED pgs=30 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:40.335 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.333+0000 7f80cdcd7640 1 --2- 172.21.15.78:0/2924227888 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f80c81293f0 0x7f80c8130970 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:40.336 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.333+0000 7f80cdcd7640 1 --2- 172.21.15.78:0/2924227888 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f80c81288b0 0x7f80c8130430 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:40.336 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.333+0000 7f80cdcd7640 1 -- 172.21.15.78:0/2924227888 >> 172.21.15.78:0/2924227888 conn(0x7f80c809c8a0 msgr2=0x7f80c806e5c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:20:40.336 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.333+0000 7f80cdcd7640 1 -- 172.21.15.78:0/2924227888 shutdown_connections 2024-09-21T09:20:40.336 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.333+0000 7f80cdcd7640 1 -- 172.21.15.78:0/2924227888 wait complete. 2024-09-21T09:20:40.346 INFO:tasks.ceph.osd.0:Restarting daemon 2024-09-21T09:20:40.346 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 0 2024-09-21T09:20:40.380 INFO:tasks.ceph.mgr.x.smithi078.stderr:2024-09-21T09:20:40.377+0000 7fda56037040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-21T09:20:40.381 INFO:tasks.ceph.osd.0:Started 2024-09-21T09:20:40.382 INFO:tasks.ceph.osd.1:Restarting daemon 2024-09-21T09:20:40.382 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 1 2024-09-21T09:20:40.384 INFO:tasks.ceph.osd.1:Started 2024-09-21T09:20:40.384 INFO:tasks.ceph.osd.2:Restarting daemon 2024-09-21T09:20:40.384 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 2 2024-09-21T09:20:40.386 INFO:tasks.ceph.osd.2:Started 2024-09-21T09:20:40.386 INFO:tasks.ceph.osd.3:Restarting daemon 2024-09-21T09:20:40.387 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 3 2024-09-21T09:20:40.389 INFO:tasks.ceph.osd.3:Started 2024-09-21T09:20:40.389 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-09-21T09:20:40.424 INFO:tasks.ceph.mgr.y.smithi078.stderr:2024-09-21T09:20:40.422+0000 7ff872721040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-21T09:20:40.467 INFO:tasks.ceph.mgr.x.smithi078.stderr:2024-09-21T09:20:40.465+0000 7fda56037040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-21T09:20:40.514 INFO:tasks.ceph.mgr.y.smithi078.stderr:2024-09-21T09:20:40.512+0000 7ff872721040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-21T09:20:40.525 INFO:tasks.ceph.mgr.x.smithi078.stderr:2024-09-21T09:20:40.522+0000 7fda56037040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-21T09:20:40.538 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.536+0000 7facf7827640 1 -- 172.21.15.78:0/1556415430 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7facf006eb00 msgr2=0x7facf006ef90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:40.538 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.536+0000 7facf7827640 1 --2- 172.21.15.78:0/1556415430 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7facf006eb00 0x7facf006ef90 secure :-1 s=READY pgs=35 cs=0 l=1 rev1=1 crypto rx=0x7face8005fd0 tx=0x7face802a6f0 comp rx=0 tx=0).stop 2024-09-21T09:20:40.539 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.536+0000 7facf7827640 1 -- 172.21.15.78:0/1556415430 shutdown_connections 2024-09-21T09:20:40.539 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.536+0000 7facf7827640 1 --2- 172.21.15.78:0/1556415430 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7facf006eb00 0x7facf006ef90 unknown :-1 s=CLOSED pgs=35 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:40.539 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.536+0000 7facf7827640 1 --2- 172.21.15.78:0/1556415430 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7facf006ff10 0x7facf006e5c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:40.539 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.536+0000 7facf7827640 1 --2- 172.21.15.78:0/1556415430 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7facf012c650 0x7facf012ca30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:40.539 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.536+0000 7facf7827640 1 -- 172.21.15.78:0/1556415430 >> 172.21.15.78:0/1556415430 conn(0x7facf009c980 msgr2=0x7facf009cd90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:20:40.539 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.536+0000 7facf7827640 1 -- 172.21.15.78:0/1556415430 shutdown_connections 2024-09-21T09:20:40.539 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.536+0000 7facf7827640 1 -- 172.21.15.78:0/1556415430 wait complete. 2024-09-21T09:20:40.541 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.537+0000 7facf7827640 1 Processor -- start 2024-09-21T09:20:40.541 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.537+0000 7facf7827640 1 -- start start 2024-09-21T09:20:40.542 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.537+0000 7facf7827640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7facf006eb00 0x7facf012fcd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:40.542 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.537+0000 7facf7827640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7facf006ff10 0x7facf0130210 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:40.542 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.537+0000 7facf7827640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7facf012c650 0x7facf0130750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:40.542 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.537+0000 7facf4dc2640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7facf006ff10 0x7facf0130210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:40.542 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.537+0000 7facf4dc2640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7facf006ff10 0x7facf0130210 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3300/0 says I am v2:172.21.15.78:35192/0 (socket says 172.21.15.78:35192) 2024-09-21T09:20:40.542 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.537+0000 7facf4dc2640 1 -- 172.21.15.78:0/402986521 learned_addr learned my addr 172.21.15.78:0/402986521 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:20:40.542 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.537+0000 7facf7827640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 v1 -- 0x7facf00712d0 con 0x7facf006ff10 2024-09-21T09:20:40.542 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.537+0000 7facf7827640 1 -- 172.21.15.78:0/402986521 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 v1 -- 0x7facf0131e40 con 0x7facf006eb00 2024-09-21T09:20:40.542 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.537+0000 7facf7827640 1 -- 172.21.15.78:0/402986521 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 v1 -- 0x7facf008d3c0 con 0x7facf012c650 2024-09-21T09:20:40.542 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.537+0000 7facf5dc4640 1 --2- 172.21.15.78:0/402986521 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7facf012c650 0x7facf0130750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:40.542 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.537+0000 7facf55c3640 1 --2- 172.21.15.78:0/402986521 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7facf006eb00 0x7facf012fcd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:40.542 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.538+0000 7facf4dc2640 1 -- 172.21.15.78:0/402986521 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7facf012c650 msgr2=0x7facf0130750 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:40.542 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.538+0000 7facf4dc2640 1 --2- 172.21.15.78:0/402986521 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7facf012c650 0x7facf0130750 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:40.542 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.538+0000 7facf4dc2640 1 -- 172.21.15.78:0/402986521 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7facf006eb00 msgr2=0x7facf012fcd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:40.543 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.538+0000 7facf4dc2640 1 --2- 172.21.15.78:0/402986521 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7facf006eb00 0x7facf012fcd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:40.543 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.538+0000 7facf4dc2640 1 -- 172.21.15.78:0/402986521 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7face8009840 con 0x7facf006ff10 2024-09-21T09:20:40.543 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.538+0000 7facf55c3640 1 --2- 172.21.15.78:0/402986521 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7facf006eb00 0x7facf012fcd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-09-21T09:20:40.543 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.538+0000 7facf4dc2640 1 --2- 172.21.15.78:0/402986521 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7facf006ff10 0x7facf0130210 secure :-1 s=READY pgs=36 cs=0 l=1 rev1=1 crypto rx=0x7face000e9c0 tx=0x7face000ee80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:20:40.543 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.538+0000 7face5ffb640 1 -- 172.21.15.78:0/402986521 <== mon.0 v2:172.21.15.78:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7face000ccc0 con 0x7facf006ff10 2024-09-21T09:20:40.543 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.538+0000 7face5ffb640 1 -- 172.21.15.78:0/402986521 <== mon.0 v2:172.21.15.78:3300/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7face000ce20 con 0x7facf006ff10 2024-09-21T09:20:40.543 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.538+0000 7face5ffb640 1 -- 172.21.15.78:0/402986521 <== mon.0 v2:172.21.15.78:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7face0004280 con 0x7facf006ff10 2024-09-21T09:20:40.543 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.538+0000 7facf7827640 1 -- 172.21.15.78:0/402986521 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7facf0132f70 con 0x7facf006ff10 2024-09-21T09:20:40.543 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.538+0000 7face5ffb640 1 -- 172.21.15.78:0/402986521 <== mon.0 v2:172.21.15.78:3300/0 4 ==== mgrmap(e 1) v1 ==== 668+0+0 (secure 0 0 0) 0x7face00035f0 con 0x7facf006ff10 2024-09-21T09:20:40.543 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.538+0000 7facf7827640 1 -- 172.21.15.78:0/402986521 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7facf01c81b0 con 0x7facf006ff10 2024-09-21T09:20:40.543 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.538+0000 7face5ffb640 1 -- 172.21.15.78:0/402986521 <== mon.0 v2:172.21.15.78:3300/0 5 ==== osd_map(6..6 src has 1..6) v4 ==== 1185+0+0 (secure 0 0 0) 0x7face0014070 con 0x7facf006ff10 2024-09-21T09:20:40.543 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.539+0000 7facf7827640 1 -- 172.21.15.78:0/402986521 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7facf006e5c0 con 0x7facf006ff10 2024-09-21T09:20:40.544 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.541+0000 7face5ffb640 1 -- 172.21.15.78:0/402986521 <== mon.0 v2:172.21.15.78:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+73245 (secure 0 0 0) 0x7face0003a30 con 0x7facf006ff10 2024-09-21T09:20:40.559 INFO:tasks.ceph.mgr.y.smithi078.stderr:2024-09-21T09:20:40.557+0000 7ff872721040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-21T09:20:40.601 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.599+0000 7facf7827640 1 -- 172.21.15.78:0/402986521 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7facf0128270 con 0x7facf006ff10 2024-09-21T09:20:40.602 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.600+0000 7face5ffb640 1 -- 172.21.15.78:0/402986521 <== mon.0 v2:172.21.15.78:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v6) v1 ==== 74+0+4152 (secure 0 0 0) 0x7face0003c80 con 0x7facf006ff10 2024-09-21T09:20:40.602 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:20:40.602 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":6,"fsid":"3633ad52-1063-4406-97d9-d53c339f43ca","created":"2024-09-21T09:20:38.248415+0000","modified":"2024-09-21T09:20:40.330925+0000","last_up_change":"0.000000","last_in_change":"2024-09-21T09:20:40.330925+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"6b18d222-0705-4495-a01c-3e2ba2ab4cff","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"2d15b0fa-66c2-4bda-b64b-d72cbbb16fb1","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"0bf75d8f-017d-488d-8a2e-c3fa3b2a29c2","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":3,"uuid":"5e3788c8-b89a-4614-b3ec-7ee1250ddbe0","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T09:20:40.603 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.601+0000 7facf7827640 1 -- 172.21.15.78:0/402986521 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7facf006ff10 msgr2=0x7facf0130210 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:40.603 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.601+0000 7facf7827640 1 --2- 172.21.15.78:0/402986521 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7facf006ff10 0x7facf0130210 secure :-1 s=READY pgs=36 cs=0 l=1 rev1=1 crypto rx=0x7face000e9c0 tx=0x7face000ee80 comp rx=0 tx=0).stop 2024-09-21T09:20:40.603 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.601+0000 7facf7827640 1 -- 172.21.15.78:0/402986521 shutdown_connections 2024-09-21T09:20:40.603 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.601+0000 7facf7827640 1 --2- 172.21.15.78:0/402986521 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7facf012c650 0x7facf0130750 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:40.603 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.601+0000 7facf7827640 1 --2- 172.21.15.78:0/402986521 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7facf006ff10 0x7facf0130210 unknown :-1 s=CLOSED pgs=36 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:40.604 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.601+0000 7facf7827640 1 --2- 172.21.15.78:0/402986521 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7facf006eb00 0x7facf012fcd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:40.604 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.601+0000 7facf7827640 1 -- 172.21.15.78:0/402986521 >> 172.21.15.78:0/402986521 conn(0x7facf009c980 msgr2=0x7facf006d070 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:20:40.604 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.601+0000 7facf7827640 1 -- 172.21.15.78:0/402986521 shutdown_connections 2024-09-21T09:20:40.604 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:40.601+0000 7facf7827640 1 -- 172.21.15.78:0/402986521 wait complete. 2024-09-21T09:20:40.613 INFO:tasks.ceph.ceph_manager.ceph:[] 2024-09-21T09:20:40.613 INFO:tasks.ceph:Waiting for OSDs to come up 2024-09-21T09:20:40.639 INFO:tasks.ceph.mgr.x.smithi078.stderr:2024-09-21T09:20:40.637+0000 7fda56037040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-21T09:20:40.660 INFO:tasks.ceph.mgr.y.smithi078.stderr:2024-09-21T09:20:40.658+0000 7ff872721040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-21T09:20:40.689 INFO:tasks.ceph.mgr.x.smithi078.stderr:2024-09-21T09:20:40.687+0000 7fda56037040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-21T09:20:40.708 INFO:tasks.ceph.mgr.y.smithi078.stderr:2024-09-21T09:20:40.706+0000 7ff872721040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-21T09:20:40.741 INFO:tasks.ceph.mgr.x.smithi078.stderr:2024-09-21T09:20:40.739+0000 7fda56037040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-21T09:20:40.757 INFO:tasks.ceph.mgr.y.smithi078.stderr:2024-09-21T09:20:40.755+0000 7ff872721040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-21T09:20:40.810 INFO:tasks.ceph.mgr.x.smithi078.stderr:2024-09-21T09:20:40.808+0000 7fda56037040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-21T09:20:40.811 INFO:tasks.ceph.mgr.y.smithi078.stderr:2024-09-21T09:20:40.809+0000 7ff872721040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-21T09:20:40.854 INFO:tasks.ceph.mgr.x.smithi078.stderr:2024-09-21T09:20:40.852+0000 7fda56037040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-21T09:20:40.855 INFO:tasks.ceph.mgr.y.smithi078.stderr:2024-09-21T09:20:40.853+0000 7ff872721040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-21T09:20:40.898 INFO:tasks.ceph.mgr.x.smithi078.stderr:2024-09-21T09:20:40.896+0000 7fda56037040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-21T09:20:40.899 INFO:tasks.ceph.mgr.y.smithi078.stderr:2024-09-21T09:20:40.897+0000 7ff872721040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-21T09:20:41.000 INFO:tasks.ceph.osd.3.smithi078.stderr:2024-09-21T09:20:40.998+0000 7f0c1a00c640 -1 Falling back to public interface 2024-09-21T09:20:41.001 INFO:tasks.ceph.osd.1.smithi078.stderr:2024-09-21T09:20:40.998+0000 7f3b43215640 -1 Falling back to public interface 2024-09-21T09:20:41.001 INFO:tasks.ceph.osd.3.smithi078.stderr:2024-09-21T09:20:40.998+0000 7f0c188e3640 -1 PosixStack listen unable to listen on v1:0.0.0.0:6800/0: (98) Address already in use 2024-09-21T09:20:41.001 INFO:tasks.ceph.osd.3.smithi078.stderr:2024-09-21T09:20:40.998+0000 7f0c188e3640 -1 PosixStack listen unable to listen on v1:0.0.0.0:6801/0: (98) Address already in use 2024-09-21T09:20:41.017 DEBUG:teuthology.orchestra.run.smithi078:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2024-09-21T09:20:41.020 INFO:tasks.ceph.osd.0.smithi078.stderr:2024-09-21T09:20:41.017+0000 7f430c40d640 -1 Falling back to public interface 2024-09-21T09:20:41.027 INFO:tasks.ceph.osd.2.smithi078.stderr:2024-09-21T09:20:41.025+0000 7f6f61f3e640 -1 Falling back to public interface 2024-09-21T09:20:41.111 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:41.108+0000 7fd52b061640 1 -- 172.21.15.78:0/58803264 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fd5241299f0 msgr2=0x7fd524125e40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:41.111 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:41.108+0000 7fd52b061640 1 --2- 172.21.15.78:0/58803264 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fd5241299f0 0x7fd524125e40 secure :-1 s=READY pgs=37 cs=0 l=1 rev1=1 crypto rx=0x7fd51c005fd0 tx=0x7fd51c02a460 comp rx=0 tx=0).stop 2024-09-21T09:20:41.111 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:41.108+0000 7fd52b061640 1 -- 172.21.15.78:0/58803264 shutdown_connections 2024-09-21T09:20:41.111 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:41.108+0000 7fd52b061640 1 --2- 172.21.15.78:0/58803264 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fd5241295b0 0x7fd524125900 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:41.111 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:41.108+0000 7fd52b061640 1 --2- 172.21.15.78:0/58803264 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fd5241299f0 0x7fd524125e40 unknown :-1 s=CLOSED pgs=37 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:41.111 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:41.108+0000 7fd52b061640 1 --2- 172.21.15.78:0/58803264 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fd52406f450 0x7fd524129070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:41.111 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:41.108+0000 7fd52b061640 1 -- 172.21.15.78:0/58803264 >> 172.21.15.78:0/58803264 conn(0x7fd5240986a0 msgr2=0x7fd524098ab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:20:41.111 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:41.108+0000 7fd52b061640 1 -- 172.21.15.78:0/58803264 shutdown_connections 2024-09-21T09:20:41.111 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:41.108+0000 7fd52b061640 1 -- 172.21.15.78:0/58803264 wait complete. 2024-09-21T09:20:41.111 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:41.108+0000 7fd52b061640 1 Processor -- start 2024-09-21T09:20:41.111 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:41.109+0000 7fd52b061640 1 -- start start 2024-09-21T09:20:41.113 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:41.109+0000 7fd52b061640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fd52406f450 0x7fd524095b00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:41.113 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:41.109+0000 7fd52b061640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fd5241295b0 0x7fd524124010 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:41.113 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:41.109+0000 7fd52b061640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fd5241299f0 0x7fd524124550 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:41.113 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:41.109+0000 7fd52b061640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd524089150 con 0x7fd5241295b0 2024-09-21T09:20:41.114 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:41.109+0000 7fd52b061640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd52408a4a0 con 0x7fd52406f450 2024-09-21T09:20:41.114 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:41.109+0000 7fd52b061640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 v1 -- 0x7fd52408ad00 con 0x7fd5241299f0 2024-09-21T09:20:41.114 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:41.109+0000 7fd523fff640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fd5241295b0 0x7fd524124010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:41.114 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:41.109+0000 7fd5295fe640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fd5241299f0 0x7fd524124550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:41.114 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:41.109+0000 7fd523fff640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fd5241295b0 0x7fd524124010 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3300/0 says I am v2:172.21.15.78:35206/0 (socket says 172.21.15.78:35206) 2024-09-21T09:20:41.114 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:41.109+0000 7fd523fff640 1 -- 172.21.15.78:0/3093217628 learned_addr learned my addr 172.21.15.78:0/3093217628 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:20:41.114 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:41.109+0000 7fd528dfd640 1 --2- 172.21.15.78:0/3093217628 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fd52406f450 0x7fd524095b00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:41.114 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:41.110+0000 7fd5295fe640 1 -- 172.21.15.78:0/3093217628 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fd52406f450 msgr2=0x7fd524095b00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:41.114 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:41.110+0000 7fd5295fe640 1 --2- 172.21.15.78:0/3093217628 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fd52406f450 0x7fd524095b00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:41.114 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:41.110+0000 7fd5295fe640 1 -- 172.21.15.78:0/3093217628 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fd5241295b0 msgr2=0x7fd524124010 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:41.114 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:41.110+0000 7fd5295fe640 1 --2- 172.21.15.78:0/3093217628 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fd5241295b0 0x7fd524124010 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:41.114 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:41.110+0000 7fd5295fe640 1 -- 172.21.15.78:0/3093217628 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd51c009840 con 0x7fd5241299f0 2024-09-21T09:20:41.114 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:41.110+0000 7fd523fff640 1 --2- 172.21.15.78:0/3093217628 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fd5241295b0 0x7fd524124010 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:20:41.115 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:41.110+0000 7fd5295fe640 1 --2- 172.21.15.78:0/3093217628 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fd5241299f0 0x7fd524124550 secure :-1 s=READY pgs=20 cs=0 l=1 rev1=1 crypto rx=0x7fd52406ac70 tx=0x7fd51c003ba0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:20:41.115 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:41.110+0000 7fd5217fa640 1 -- 172.21.15.78:0/3093217628 <== mon.2 v2:172.21.15.78:3302/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd51c03b820 con 0x7fd5241299f0 2024-09-21T09:20:41.115 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:41.110+0000 7fd5217fa640 1 -- 172.21.15.78:0/3093217628 <== mon.2 v2:172.21.15.78:3302/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7fd51c03ba70 con 0x7fd5241299f0 2024-09-21T09:20:41.115 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:41.110+0000 7fd52b061640 1 -- 172.21.15.78:0/3093217628 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd524089760 con 0x7fd5241299f0 2024-09-21T09:20:41.115 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:41.110+0000 7fd5217fa640 1 -- 172.21.15.78:0/3093217628 <== mon.2 v2:172.21.15.78:3302/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd51c03be80 con 0x7fd5241299f0 2024-09-21T09:20:41.115 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:41.110+0000 7fd52b061640 1 -- 172.21.15.78:0/3093217628 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd524124c70 con 0x7fd5241299f0 2024-09-21T09:20:41.115 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:41.110+0000 7fd5217fa640 1 -- 172.21.15.78:0/3093217628 <== mon.2 v2:172.21.15.78:3302/0 4 ==== mgrmap(e 1) v1 ==== 668+0+0 (secure 0 0 0) 0x7fd51c038050 con 0x7fd5241299f0 2024-09-21T09:20:41.115 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:41.110+0000 7fd5217fa640 1 -- 172.21.15.78:0/3093217628 <== mon.2 v2:172.21.15.78:3302/0 5 ==== osd_map(6..6 src has 1..6) v4 ==== 1185+0+0 (secure 0 0 0) 0x7fd51c0042a0 con 0x7fd5241299f0 2024-09-21T09:20:41.115 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:41.111+0000 7fd52b061640 1 -- 172.21.15.78:0/3093217628 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd52406c860 con 0x7fd5241299f0 2024-09-21T09:20:41.115 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:41.113+0000 7fd5217fa640 1 -- 172.21.15.78:0/3093217628 <== mon.2 v2:172.21.15.78:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+73245 (secure 0 0 0) 0x7fd51c065e30 con 0x7fd5241299f0 2024-09-21T09:20:41.134 INFO:tasks.ceph.mgr.x.smithi078.stderr:2024-09-21T09:20:41.132+0000 7fda56037040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-21T09:20:41.134 INFO:tasks.ceph.mgr.y.smithi078.stderr:2024-09-21T09:20:41.132+0000 7ff872721040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-21T09:20:41.139 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:41.137+0000 7fd5217fa640 1 -- 172.21.15.78:0/3093217628 <== mon.2 v2:172.21.15.78:3302/0 7 ==== mgrmap(e 2) v1 ==== 27607+0+0 (secure 0 0 0) 0x7fd51c004500 con 0x7fd5241299f0 2024-09-21T09:20:41.182 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:41.179+0000 7fd52b061640 1 -- 172.21.15.78:0/3093217628 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fd524067670 con 0x7fd5241299f0 2024-09-21T09:20:41.182 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:41.180+0000 7fd5217fa640 1 -- 172.21.15.78:0/3093217628 <== mon.2 v2:172.21.15.78:3302/0 8 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v6) v1 ==== 74+0+4152 (secure 0 0 0) 0x7fd51c055050 con 0x7fd5241299f0 2024-09-21T09:20:41.183 INFO:teuthology.misc.health.smithi078.stdout: 2024-09-21T09:20:41.183 INFO:teuthology.misc.health.smithi078.stdout:{"epoch":6,"fsid":"3633ad52-1063-4406-97d9-d53c339f43ca","created":"2024-09-21T09:20:38.248415+0000","modified":"2024-09-21T09:20:40.330925+0000","last_up_change":"0.000000","last_in_change":"2024-09-21T09:20:40.330925+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"6b18d222-0705-4495-a01c-3e2ba2ab4cff","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"2d15b0fa-66c2-4bda-b64b-d72cbbb16fb1","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"0bf75d8f-017d-488d-8a2e-c3fa3b2a29c2","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":3,"uuid":"5e3788c8-b89a-4614-b3ec-7ee1250ddbe0","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T09:20:41.183 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:41.181+0000 7fd52b061640 1 -- 172.21.15.78:0/3093217628 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fd5241299f0 msgr2=0x7fd524124550 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:41.184 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:41.181+0000 7fd52b061640 1 --2- 172.21.15.78:0/3093217628 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fd5241299f0 0x7fd524124550 secure :-1 s=READY pgs=20 cs=0 l=1 rev1=1 crypto rx=0x7fd52406ac70 tx=0x7fd51c003ba0 comp rx=0 tx=0).stop 2024-09-21T09:20:41.184 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:41.181+0000 7fd52b061640 1 -- 172.21.15.78:0/3093217628 shutdown_connections 2024-09-21T09:20:41.184 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:41.181+0000 7fd52b061640 1 --2- 172.21.15.78:0/3093217628 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fd5241299f0 0x7fd524124550 unknown :-1 s=CLOSED pgs=20 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:41.184 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:41.181+0000 7fd52b061640 1 --2- 172.21.15.78:0/3093217628 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fd5241295b0 0x7fd524124010 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:41.184 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:41.181+0000 7fd52b061640 1 --2- 172.21.15.78:0/3093217628 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fd52406f450 0x7fd524095b00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:41.184 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:41.181+0000 7fd52b061640 1 -- 172.21.15.78:0/3093217628 >> 172.21.15.78:0/3093217628 conn(0x7fd5240986a0 msgr2=0x7fd52406ca70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:20:41.184 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:41.181+0000 7fd52b061640 1 -- 172.21.15.78:0/3093217628 shutdown_connections 2024-09-21T09:20:41.184 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:41.181+0000 7fd52b061640 1 -- 172.21.15.78:0/3093217628 wait complete. 2024-09-21T09:20:41.192 DEBUG:teuthology.misc:0 of 4 OSDs are up 2024-09-21T09:20:42.145 INFO:tasks.ceph.mgr.x.smithi078.stderr:2024-09-21T09:20:42.143+0000 7fda3eff3640 -1 mgr.server handle_report got status from non-daemon mon.c 2024-09-21T09:20:42.146 INFO:tasks.ceph.mgr.x.smithi078.stderr:2024-09-21T09:20:42.144+0000 7fda3eff3640 -1 mgr.server handle_report got status from non-daemon mon.b 2024-09-21T09:20:44.443 INFO:tasks.ceph.osd.3.smithi078.stderr:2024-09-21T09:20:44.441+0000 7f0c1a00c640 -1 osd.3 0 log_to_monitors true 2024-09-21T09:20:44.471 INFO:tasks.ceph.osd.1.smithi078.stderr:2024-09-21T09:20:44.470+0000 7f3b43215640 -1 osd.1 0 log_to_monitors true 2024-09-21T09:20:44.511 INFO:tasks.ceph.osd.0.smithi078.stderr:2024-09-21T09:20:44.510+0000 7f430c40d640 -1 osd.0 0 log_to_monitors true 2024-09-21T09:20:44.541 INFO:tasks.ceph.osd.2.smithi078.stderr:2024-09-21T09:20:44.539+0000 7f6f61f3e640 -1 osd.2 0 log_to_monitors true 2024-09-21T09:20:47.595 DEBUG:teuthology.orchestra.run.smithi078:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2024-09-21T09:20:47.718 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:47.716+0000 7f164aa2b640 1 -- 172.21.15.78:0/2928317455 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f164406d090 msgr2=0x7f164406d470 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:47.718 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:47.716+0000 7f164aa2b640 1 --2- 172.21.15.78:0/2928317455 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f164406d090 0x7f164406d470 secure :-1 s=READY pgs=38 cs=0 l=1 rev1=1 crypto rx=0x7f1634009860 tx=0x7f163402a3d0 comp rx=0 tx=0).stop 2024-09-21T09:20:47.718 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:47.717+0000 7f164aa2b640 1 -- 172.21.15.78:0/2928317455 shutdown_connections 2024-09-21T09:20:47.719 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:47.717+0000 7f164aa2b640 1 --2- 172.21.15.78:0/2928317455 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f164406d9b0 0x7f164412c250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:47.719 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:47.717+0000 7f164aa2b640 1 --2- 172.21.15.78:0/2928317455 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f164406d090 0x7f164406d470 unknown :-1 s=CLOSED pgs=38 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:47.719 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:47.717+0000 7f164aa2b640 1 -- 172.21.15.78:0/2928317455 >> 172.21.15.78:0/2928317455 conn(0x7f164409ba30 msgr2=0x7f164409be40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:20:47.719 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:47.717+0000 7f164aa2b640 1 -- 172.21.15.78:0/2928317455 shutdown_connections 2024-09-21T09:20:47.719 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:47.717+0000 7f164aa2b640 1 -- 172.21.15.78:0/2928317455 wait complete. 2024-09-21T09:20:47.719 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:47.718+0000 7f164aa2b640 1 Processor -- start 2024-09-21T09:20:47.720 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:47.719+0000 7f164aa2b640 1 -- start start 2024-09-21T09:20:47.723 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:47.722+0000 7f164aa2b640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f164406d090 0x7f164412dab0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:47.725 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:47.723+0000 7f1643fff640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f164406d090 0x7f164412dab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:47.725 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:47.723+0000 7f1643fff640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f164406d090 0x7f164412dab0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3300/0 says I am v2:172.21.15.78:59842/0 (socket says 172.21.15.78:59842) 2024-09-21T09:20:47.725 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:47.723+0000 7f1643fff640 1 -- 172.21.15.78:0/1686585257 learned_addr learned my addr 172.21.15.78:0/1686585257 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:20:47.725 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:47.724+0000 7f164aa2b640 1 --2- 172.21.15.78:0/1686585257 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f164406d9b0 0x7f164412dff0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:47.726 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:47.724+0000 7f164aa2b640 1 --2- 172.21.15.78:0/1686585257 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f164412c790 0x7f164412e530 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:47.726 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:47.724+0000 7f16437fe640 1 --2- 172.21.15.78:0/1686585257 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f164406d9b0 0x7f164412dff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:47.726 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:47.724+0000 7f164aa2b640 1 -- 172.21.15.78:0/1686585257 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1644130eb0 con 0x7f164406d090 2024-09-21T09:20:47.726 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:47.724+0000 7f1648fc8640 1 --2- 172.21.15.78:0/1686585257 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f164412c790 0x7f164412e530 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:47.726 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:47.725+0000 7f164aa2b640 1 -- 172.21.15.78:0/1686585257 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1644131280 con 0x7f164412c790 2024-09-21T09:20:47.726 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:47.725+0000 7f164aa2b640 1 -- 172.21.15.78:0/1686585257 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 v1 -- 0x7f164412fe20 con 0x7f164406d9b0 2024-09-21T09:20:47.727 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:47.726+0000 7f16437fe640 1 -- 172.21.15.78:0/1686585257 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f164412c790 msgr2=0x7f164412e530 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:47.727 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:47.726+0000 7f16437fe640 1 --2- 172.21.15.78:0/1686585257 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f164412c790 0x7f164412e530 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:47.727 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:47.726+0000 7f16437fe640 1 -- 172.21.15.78:0/1686585257 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f164406d090 msgr2=0x7f164412dab0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:47.728 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:47.726+0000 7f16437fe640 1 --2- 172.21.15.78:0/1686585257 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f164406d090 0x7f164412dab0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:47.728 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:47.726+0000 7f16437fe640 1 -- 172.21.15.78:0/1686585257 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1638009490 con 0x7f164406d9b0 2024-09-21T09:20:47.728 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:47.726+0000 7f1643fff640 1 --2- 172.21.15.78:0/1686585257 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f164406d090 0x7f164412dab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:20:47.728 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:47.726+0000 7f1648fc8640 1 --2- 172.21.15.78:0/1686585257 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f164412c790 0x7f164412e530 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-09-21T09:20:47.728 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:47.726+0000 7f16437fe640 1 --2- 172.21.15.78:0/1686585257 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f164406d9b0 0x7f164412dff0 secure :-1 s=READY pgs=27 cs=0 l=1 rev1=1 crypto rx=0x7f163800e9c0 tx=0x7f163800ee80 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:20:47.728 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:47.727+0000 7f1640ff9640 1 -- 172.21.15.78:0/1686585257 <== mon.2 v2:172.21.15.78:3302/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f163800cd80 con 0x7f164406d9b0 2024-09-21T09:20:47.729 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:47.727+0000 7f164aa2b640 1 -- 172.21.15.78:0/1686585257 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1634009490 con 0x7f164406d9b0 2024-09-21T09:20:47.729 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:47.727+0000 7f164aa2b640 1 -- 172.21.15.78:0/1686585257 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1644134000 con 0x7f164406d9b0 2024-09-21T09:20:47.729 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:47.727+0000 7f1640ff9640 1 -- 172.21.15.78:0/1686585257 <== mon.2 v2:172.21.15.78:3302/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7f1638010430 con 0x7f164406d9b0 2024-09-21T09:20:47.729 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:47.727+0000 7f1640ff9640 1 -- 172.21.15.78:0/1686585257 <== mon.2 v2:172.21.15.78:3302/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f163800cd80 con 0x7f164406d9b0 2024-09-21T09:20:47.729 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:47.728+0000 7f164aa2b640 1 -- 172.21.15.78:0/1686585257 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1624005250 con 0x7f164406d9b0 2024-09-21T09:20:47.730 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:47.729+0000 7f1640ff9640 1 -- 172.21.15.78:0/1686585257 <== mon.2 v2:172.21.15.78:3302/0 4 ==== mgrmap(e 4) v1 ==== 55156+0+0 (secure 0 0 0) 0x7f16380105c0 con 0x7f164406d9b0 2024-09-21T09:20:47.731 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:47.729+0000 7f1640ff9640 1 -- 172.21.15.78:0/1686585257 <== mon.2 v2:172.21.15.78:3302/0 5 ==== osd_map(8..8 src has 1..8) v4 ==== 1513+0+0 (secure 0 0 0) 0x7f1638014070 con 0x7f164406d9b0 2024-09-21T09:20:47.737 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:47.736+0000 7f1640ff9640 1 -- 172.21.15.78:0/1686585257 <== mon.2 v2:172.21.15.78:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+146310 (secure 0 0 0) 0x7f163803ab70 con 0x7f164406d9b0 2024-09-21T09:20:47.997 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:47.996+0000 7f164aa2b640 1 -- 172.21.15.78:0/1686585257 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f1624005500 con 0x7f164406d9b0 2024-09-21T09:20:47.998 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:47.996+0000 7f1640ff9640 1 -- 172.21.15.78:0/1686585257 <== mon.2 v2:172.21.15.78:3302/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v8) v1 ==== 74+0+4152 (secure 0 0 0) 0x7f163803ab70 con 0x7f164406d9b0 2024-09-21T09:20:47.998 INFO:teuthology.misc.health.smithi078.stdout: 2024-09-21T09:20:47.998 INFO:teuthology.misc.health.smithi078.stdout:{"epoch":8,"fsid":"3633ad52-1063-4406-97d9-d53c339f43ca","created":"2024-09-21T09:20:38.248415+0000","modified":"2024-09-21T09:20:46.151307+0000","last_up_change":"0.000000","last_in_change":"2024-09-21T09:20:40.330925+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"6b18d222-0705-4495-a01c-3e2ba2ab4cff","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"2d15b0fa-66c2-4bda-b64b-d72cbbb16fb1","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"0bf75d8f-017d-488d-8a2e-c3fa3b2a29c2","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":3,"uuid":"5e3788c8-b89a-4614-b3ec-7ee1250ddbe0","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T09:20:48.001 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:47.999+0000 7f164aa2b640 1 -- 172.21.15.78:0/1686585257 >> v1:172.21.15.78:6816/2602406893 conn(0x7f1630047530 legacy=0x7f16300499f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:48.001 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:47.999+0000 7f164aa2b640 1 -- 172.21.15.78:0/1686585257 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f164406d9b0 msgr2=0x7f164412dff0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:48.001 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:47.999+0000 7f164aa2b640 1 --2- 172.21.15.78:0/1686585257 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f164406d9b0 0x7f164412dff0 secure :-1 s=READY pgs=27 cs=0 l=1 rev1=1 crypto rx=0x7f163800e9c0 tx=0x7f163800ee80 comp rx=0 tx=0).stop 2024-09-21T09:20:48.001 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:47.999+0000 7f164aa2b640 1 -- 172.21.15.78:0/1686585257 shutdown_connections 2024-09-21T09:20:48.001 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:47.999+0000 7f164aa2b640 1 --2- 172.21.15.78:0/1686585257 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f164412c790 0x7f164412e530 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:48.001 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:47.999+0000 7f164aa2b640 1 --2- 172.21.15.78:0/1686585257 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f164406d9b0 0x7f164412dff0 unknown :-1 s=CLOSED pgs=27 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:48.001 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:47.999+0000 7f164aa2b640 1 --2- 172.21.15.78:0/1686585257 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f164406d090 0x7f164412dab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:48.001 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:47.999+0000 7f164aa2b640 1 -- 172.21.15.78:0/1686585257 >> 172.21.15.78:0/1686585257 conn(0x7f164409ba30 msgr2=0x7f164406fdc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:20:48.001 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:47.999+0000 7f164aa2b640 1 -- 172.21.15.78:0/1686585257 shutdown_connections 2024-09-21T09:20:48.001 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:47.999+0000 7f164aa2b640 1 -- 172.21.15.78:0/1686585257 wait complete. 2024-09-21T09:20:48.014 DEBUG:teuthology.misc:0 of 4 OSDs are up 2024-09-21T09:20:48.273 INFO:tasks.ceph.osd.0.smithi078.stderr:2024-09-21T09:20:48.272+0000 7f4308e61640 -1 osd.0 0 waiting for initial osdmap 2024-09-21T09:20:48.283 INFO:tasks.ceph.osd.0.smithi078.stderr:2024-09-21T09:20:48.282+0000 7f42ffc59640 -1 osd.0 8 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:20:48.303 INFO:tasks.ceph.osd.1.smithi078.stderr:2024-09-21T09:20:48.302+0000 7f3b3e3db640 -1 osd.1 0 waiting for initial osdmap 2024-09-21T09:20:48.306 INFO:tasks.ceph.osd.1.smithi078.stderr:2024-09-21T09:20:48.305+0000 7f3b36a0e640 -1 osd.1 8 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:20:48.306 INFO:tasks.ceph.osd.3.smithi078.stderr:2024-09-21T09:20:48.305+0000 7f0c150a4640 -1 osd.3 0 waiting for initial osdmap 2024-09-21T09:20:48.314 INFO:tasks.ceph.osd.3.smithi078.stderr:2024-09-21T09:20:48.313+0000 7f0c0d6d7640 -1 osd.3 8 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:20:48.333 INFO:tasks.ceph.osd.2.smithi078.stderr:2024-09-21T09:20:48.332+0000 7f6f5d6a8640 -1 osd.2 0 waiting for initial osdmap 2024-09-21T09:20:48.337 INFO:tasks.ceph.osd.2.smithi078.stderr:2024-09-21T09:20:48.335+0000 7f6f55cdb640 -1 osd.2 8 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:20:54.416 DEBUG:teuthology.orchestra.run.smithi078:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2024-09-21T09:20:54.509 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:54.508+0000 7fefd32e5640 1 -- 172.21.15.78:0/831936065 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fefcc095e60 msgr2=0x7fefcc0962d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:54.510 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:54.508+0000 7fefd32e5640 1 --2- 172.21.15.78:0/831936065 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fefcc095e60 0x7fefcc0962d0 secure :-1 s=READY pgs=41 cs=0 l=1 rev1=1 crypto rx=0x7fefc4005fd0 tx=0x7fefc402a8e0 comp rx=0 tx=0).stop 2024-09-21T09:20:54.510 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:54.508+0000 7fefd32e5640 1 -- 172.21.15.78:0/831936065 shutdown_connections 2024-09-21T09:20:54.510 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:54.508+0000 7fefd32e5640 1 --2- 172.21.15.78:0/831936065 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fefcc129460 0x7fefcc095920 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:54.510 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:54.508+0000 7fefd32e5640 1 --2- 172.21.15.78:0/831936065 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fefcc095e60 0x7fefcc0962d0 unknown :-1 s=CLOSED pgs=41 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:54.510 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:54.508+0000 7fefd32e5640 1 --2- 172.21.15.78:0/831936065 >> v2:172.21.15.78:3300/0 conn(0x7fefcc128b40 0x7fefcc128f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:54.510 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:54.508+0000 7fefd32e5640 1 -- 172.21.15.78:0/831936065 >> 172.21.15.78:0/831936065 conn(0x7fefcc0986e0 msgr2=0x7fefcc098af0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:20:54.510 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:54.508+0000 7fefd32e5640 1 -- 172.21.15.78:0/831936065 shutdown_connections 2024-09-21T09:20:54.510 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:54.508+0000 7fefd32e5640 1 -- 172.21.15.78:0/831936065 wait complete. 2024-09-21T09:20:54.512 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:54.509+0000 7fefd32e5640 1 Processor -- start 2024-09-21T09:20:54.512 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:54.509+0000 7fefd32e5640 1 -- start start 2024-09-21T09:20:54.513 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:54.509+0000 7fefd32e5640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fefcc095e60 0x7fefcc1bfd30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:54.513 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:54.509+0000 7fefd32e5640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fefcc128b40 0x7fefcc1c0270 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:54.513 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:54.509+0000 7fefd32e5640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fefcc129460 0x7fefcc120220 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:54.513 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:54.509+0000 7fefd32e5640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fefcc121d40 con 0x7fefcc128b40 2024-09-21T09:20:54.513 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:54.509+0000 7fefd32e5640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fefcc123670 con 0x7fefcc095e60 2024-09-21T09:20:54.513 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:54.509+0000 7fefd32e5640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 v1 -- 0x7fefcc096c90 con 0x7fefcc129460 2024-09-21T09:20:54.513 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:54.509+0000 7fefd1081640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fefcc095e60 0x7fefcc1bfd30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:54.513 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:54.509+0000 7fefd1081640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fefcc095e60 0x7fefcc1bfd30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3301/0 says I am v2:172.21.15.78:60996/0 (socket says 172.21.15.78:60996) 2024-09-21T09:20:54.513 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:54.509+0000 7fefd1081640 1 -- 172.21.15.78:0/4093879455 learned_addr learned my addr 172.21.15.78:0/4093879455 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:20:54.513 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:54.509+0000 7fefd1882640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fefcc129460 0x7fefcc120220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:54.513 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:54.509+0000 7fefd0880640 1 --2- 172.21.15.78:0/4093879455 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fefcc128b40 0x7fefcc1c0270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:54.513 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:54.509+0000 7fefd1081640 1 -- 172.21.15.78:0/4093879455 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fefcc129460 msgr2=0x7fefcc120220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:54.513 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:54.509+0000 7fefd1081640 1 --2- 172.21.15.78:0/4093879455 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fefcc129460 0x7fefcc120220 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:54.514 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:54.509+0000 7fefd1081640 1 -- 172.21.15.78:0/4093879455 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fefcc128b40 msgr2=0x7fefcc1c0270 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:54.514 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:54.509+0000 7fefd1081640 1 --2- 172.21.15.78:0/4093879455 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fefcc128b40 0x7fefcc1c0270 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:54.514 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:54.509+0000 7fefd1081640 1 -- 172.21.15.78:0/4093879455 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fefbc009490 con 0x7fefcc095e60 2024-09-21T09:20:54.514 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:54.510+0000 7fefd1081640 1 --2- 172.21.15.78:0/4093879455 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fefcc095e60 0x7fefcc1bfd30 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7fefbc00ea60 tx=0x7fefbc00ef20 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:20:54.514 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:54.510+0000 7fefc1ffb640 1 -- 172.21.15.78:0/4093879455 <== mon.1 v2:172.21.15.78:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fefbc00cde0 con 0x7fefcc095e60 2024-09-21T09:20:54.514 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:54.510+0000 7fefc1ffb640 1 -- 172.21.15.78:0/4093879455 <== mon.1 v2:172.21.15.78:3301/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7fefbc004020 con 0x7fefcc095e60 2024-09-21T09:20:54.514 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:54.510+0000 7fefd32e5640 1 -- 172.21.15.78:0/4093879455 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fefc8009490 con 0x7fefcc095e60 2024-09-21T09:20:54.514 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:54.510+0000 7fefc1ffb640 1 -- 172.21.15.78:0/4093879455 <== mon.1 v2:172.21.15.78:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fefbc004410 con 0x7fefcc095e60 2024-09-21T09:20:54.514 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:54.510+0000 7fefd32e5640 1 -- 172.21.15.78:0/4093879455 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fefc4009900 con 0x7fefcc095e60 2024-09-21T09:20:54.514 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:54.511+0000 7fefc1ffb640 1 -- 172.21.15.78:0/4093879455 <== mon.1 v2:172.21.15.78:3301/0 4 ==== mgrmap(e 5) v1 ==== 55212+0+0 (secure 0 0 0) 0x7fefbc0035f0 con 0x7fefcc095e60 2024-09-21T09:20:54.514 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:54.511+0000 7fefd32e5640 1 -- 172.21.15.78:0/4093879455 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fefcc123b60 con 0x7fefcc095e60 2024-09-21T09:20:54.514 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:54.511+0000 7fefc1ffb640 1 -- 172.21.15.78:0/4093879455 <== mon.1 v2:172.21.15.78:3301/0 5 ==== osd_map(13..13 src has 1..13) v4 ==== 2459+0+0 (secure 0 0 0) 0x7fefbc014070 con 0x7fefcc095e60 2024-09-21T09:20:54.516 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:54.515+0000 7fefc1ffb640 1 -- 172.21.15.78:0/4093879455 <== mon.1 v2:172.21.15.78:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+146310 (secure 0 0 0) 0x7fefcc123b60 con 0x7fefcc095e60 2024-09-21T09:20:54.649 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:54.648+0000 7fefd32e5640 1 -- 172.21.15.78:0/4093879455 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fefcc123b60 con 0x7fefcc095e60 2024-09-21T09:20:54.650 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:54.649+0000 7fefc1ffb640 1 -- 172.21.15.78:0/4093879455 <== mon.1 v2:172.21.15.78:3301/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v13) v1 ==== 74+0+6957 (secure 0 0 0) 0x7fefcc123b60 con 0x7fefcc095e60 2024-09-21T09:20:54.650 INFO:teuthology.misc.health.smithi078.stdout: 2024-09-21T09:20:54.650 INFO:teuthology.misc.health.smithi078.stdout:{"epoch":13,"fsid":"3633ad52-1063-4406-97d9-d53c339f43ca","created":"2024-09-21T09:20:38.248415+0000","modified":"2024-09-21T09:20:53.287884+0000","last_up_change":"2024-09-21T09:20:49.276020+0000","last_in_change":"2024-09-21T09:20:40.330925+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T09:20:51.169034+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"6b18d222-0705-4495-a01c-3e2ba2ab4cff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6808","nonce":1449157914}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6809","nonce":1449157914}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6811","nonce":1449157914}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6810","nonce":1449157914}]},"public_addr":"172.21.15.78:6808/1449157914","cluster_addr":"172.21.15.78:6809/1449157914","heartbeat_back_addr":"172.21.15.78:6811/1449157914","heartbeat_front_addr":"172.21.15.78:6810/1449157914","state":["exists","up"]},{"osd":1,"uuid":"2d15b0fa-66c2-4bda-b64b-d72cbbb16fb1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":11,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6800","nonce":3190796132}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6801","nonce":3190796132}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6805","nonce":3190796132}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6803","nonce":3190796132}]},"public_addr":"172.21.15.78:6800/3190796132","cluster_addr":"172.21.15.78:6801/3190796132","heartbeat_back_addr":"172.21.15.78:6805/3190796132","heartbeat_front_addr":"172.21.15.78:6803/3190796132","state":["exists","up"]},{"osd":2,"uuid":"0bf75d8f-017d-488d-8a2e-c3fa3b2a29c2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6812","nonce":3442117044}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6813","nonce":3442117044}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6815","nonce":3442117044}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6814","nonce":3442117044}]},"public_addr":"172.21.15.78:6812/3442117044","cluster_addr":"172.21.15.78:6813/3442117044","heartbeat_back_addr":"172.21.15.78:6815/3442117044","heartbeat_front_addr":"172.21.15.78:6814/3442117044","state":["exists","up"]},{"osd":3,"uuid":"5e3788c8-b89a-4614-b3ec-7ee1250ddbe0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6802","nonce":4276488585}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6804","nonce":4276488585}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6807","nonce":4276488585}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6806","nonce":4276488585}]},"public_addr":"172.21.15.78:6802/4276488585","cluster_addr":"172.21.15.78:6804/4276488585","heartbeat_back_addr":"172.21.15.78:6807/4276488585","heartbeat_front_addr":"172.21.15.78:6806/4276488585","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:20:45.499319+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:20:45.489695+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:20:45.541959+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:20:45.476915+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T09:20:54.652 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:54.650+0000 7fefd32e5640 1 -- 172.21.15.78:0/4093879455 >> v1:172.21.15.78:6816/2602406893 conn(0x7fefb0047560 legacy=0x7fefb0049a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:54.652 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:54.650+0000 7fefd32e5640 1 -- 172.21.15.78:0/4093879455 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fefcc095e60 msgr2=0x7fefcc1bfd30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:54.652 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:54.650+0000 7fefd32e5640 1 --2- 172.21.15.78:0/4093879455 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fefcc095e60 0x7fefcc1bfd30 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7fefbc00ea60 tx=0x7fefbc00ef20 comp rx=0 tx=0).stop 2024-09-21T09:20:54.652 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:54.650+0000 7fefd32e5640 1 -- 172.21.15.78:0/4093879455 shutdown_connections 2024-09-21T09:20:54.652 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:54.650+0000 7fefd32e5640 1 --2- 172.21.15.78:0/4093879455 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fefcc129460 0x7fefcc120220 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:54.652 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:54.650+0000 7fefd32e5640 1 --2- 172.21.15.78:0/4093879455 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fefcc128b40 0x7fefcc1c0270 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:54.652 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:54.650+0000 7fefd32e5640 1 --2- 172.21.15.78:0/4093879455 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fefcc095e60 0x7fefcc1bfd30 unknown :-1 s=CLOSED pgs=42 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:54.653 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:54.650+0000 7fefd32e5640 1 -- 172.21.15.78:0/4093879455 >> 172.21.15.78:0/4093879455 conn(0x7fefcc0986e0 msgr2=0x7fefcc06bae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:20:54.653 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:54.650+0000 7fefd32e5640 1 -- 172.21.15.78:0/4093879455 shutdown_connections 2024-09-21T09:20:54.653 INFO:teuthology.misc.health.smithi078.stderr:2024-09-21T09:20:54.651+0000 7fefd32e5640 1 -- 172.21.15.78:0/4093879455 wait complete. 2024-09-21T09:20:54.661 DEBUG:teuthology.misc:4 of 4 OSDs are up 2024-09-21T09:20:54.661 INFO:tasks.ceph:Creating RBD pool 2024-09-21T09:20:54.661 DEBUG:teuthology.orchestra.run.smithi078:> sudo ceph --cluster ceph osd pool create rbd 8 2024-09-21T09:20:54.759 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:54.758+0000 7fb706b6c640 1 -- 172.21.15.78:0/1390387418 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fb70012e5d0 msgr2=0x7fb7001309c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:54.759 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:54.758+0000 7fb706b6c640 1 --2- 172.21.15.78:0/1390387418 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fb70012e5d0 0x7fb7001309c0 secure :-1 s=READY pgs=50 cs=0 l=1 rev1=1 crypto rx=0x7fb6e8009830 tx=0x7fb6e802a3f0 comp rx=0 tx=0).stop 2024-09-21T09:20:54.759 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:54.758+0000 7fb706b6c640 1 -- 172.21.15.78:0/1390387418 shutdown_connections 2024-09-21T09:20:54.760 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:54.758+0000 7fb706b6c640 1 --2- 172.21.15.78:0/1390387418 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fb70012e5d0 0x7fb7001309c0 unknown :-1 s=CLOSED pgs=50 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:54.760 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:54.758+0000 7fb706b6c640 1 --2- 172.21.15.78:0/1390387418 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fb7000710c0 0x7fb70012e090 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:54.760 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:54.758+0000 7fb706b6c640 1 -- 172.21.15.78:0/1390387418 >> 172.21.15.78:0/1390387418 conn(0x7fb700097590 msgr2=0x7fb7000979a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:20:54.760 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:54.758+0000 7fb706b6c640 1 -- 172.21.15.78:0/1390387418 shutdown_connections 2024-09-21T09:20:54.760 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:54.758+0000 7fb706b6c640 1 -- 172.21.15.78:0/1390387418 wait complete. 2024-09-21T09:20:54.762 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:54.758+0000 7fb706b6c640 1 Processor -- start 2024-09-21T09:20:54.762 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:54.758+0000 7fb706b6c640 1 -- start start 2024-09-21T09:20:54.762 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:54.759+0000 7fb706b6c640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fb7000710c0 0x7fb700123410 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:54.762 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:54.759+0000 7fb706b6c640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fb700129130 0x7fb700123950 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:54.762 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:54.759+0000 7fb706b6c640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fb70012e5d0 0x7fb700123e90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:54.762 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:54.759+0000 7fb706b6c640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb700096ea0 con 0x7fb70012e5d0 2024-09-21T09:20:54.762 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:54.759+0000 7fb706b6c640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb700124820 con 0x7fb700129130 2024-09-21T09:20:54.763 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:54.759+0000 7fb706b6c640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 v1 -- 0x7fb70006e3e0 con 0x7fb7000710c0 2024-09-21T09:20:54.763 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:54.759+0000 7fb704908640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fb7000710c0 0x7fb700123410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:54.763 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:54.759+0000 7fb705109640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fb70012e5d0 0x7fb700123e90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:54.763 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:54.759+0000 7fb6fffff640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fb700129130 0x7fb700123950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:54.763 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:54.759+0000 7fb704908640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fb7000710c0 0x7fb700123410 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3302/0 says I am v2:172.21.15.78:35720/0 (socket says 172.21.15.78:35720) 2024-09-21T09:20:54.763 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:54.759+0000 7fb705109640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fb70012e5d0 0x7fb700123e90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3300/0 says I am v2:172.21.15.78:37456/0 (socket says 172.21.15.78:37456) 2024-09-21T09:20:54.763 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:54.759+0000 7fb704908640 1 -- 172.21.15.78:0/3779698690 learned_addr learned my addr 172.21.15.78:0/3779698690 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:20:54.763 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:54.759+0000 7fb6fffff640 1 -- 172.21.15.78:0/3779698690 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fb7000710c0 msgr2=0x7fb700123410 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:54.763 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:54.759+0000 7fb6fffff640 1 --2- 172.21.15.78:0/3779698690 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fb7000710c0 0x7fb700123410 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:54.763 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:54.759+0000 7fb6fffff640 1 -- 172.21.15.78:0/3779698690 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fb70012e5d0 msgr2=0x7fb700123e90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:54.763 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:54.759+0000 7fb6fffff640 1 --2- 172.21.15.78:0/3779698690 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fb70012e5d0 0x7fb700123e90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:54.763 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:54.759+0000 7fb6fffff640 1 -- 172.21.15.78:0/3779698690 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb6f0009840 con 0x7fb700129130 2024-09-21T09:20:54.763 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:54.759+0000 7fb704908640 1 --2- 172.21.15.78:0/3779698690 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fb7000710c0 0x7fb700123410 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:20:54.764 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:54.760+0000 7fb6fffff640 1 --2- 172.21.15.78:0/3779698690 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fb700129130 0x7fb700123950 secure :-1 s=READY pgs=43 cs=0 l=1 rev1=1 crypto rx=0x7fb6f0005fd0 tx=0x7fb6f000c6a0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:20:54.764 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:54.760+0000 7fb6fd7fa640 1 -- 172.21.15.78:0/3779698690 <== mon.1 v2:172.21.15.78:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb6f0010630 con 0x7fb700129130 2024-09-21T09:20:54.764 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:54.760+0000 7fb6fd7fa640 1 -- 172.21.15.78:0/3779698690 <== mon.1 v2:172.21.15.78:3301/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7fb6f0010790 con 0x7fb700129130 2024-09-21T09:20:54.764 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:54.760+0000 7fb706b6c640 1 -- 172.21.15.78:0/3779698690 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb6e8009490 con 0x7fb700129130 2024-09-21T09:20:54.764 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:54.760+0000 7fb6fd7fa640 1 -- 172.21.15.78:0/3779698690 <== mon.1 v2:172.21.15.78:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb6f0010b80 con 0x7fb700129130 2024-09-21T09:20:54.764 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:54.760+0000 7fb706b6c640 1 -- 172.21.15.78:0/3779698690 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb700125180 con 0x7fb700129130 2024-09-21T09:20:54.764 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:54.761+0000 7fb6fd7fa640 1 -- 172.21.15.78:0/3779698690 <== mon.1 v2:172.21.15.78:3301/0 4 ==== mgrmap(e 5) v1 ==== 55212+0+0 (secure 0 0 0) 0x7fb6f0010ce0 con 0x7fb700129130 2024-09-21T09:20:54.764 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:54.761+0000 7fb706b6c640 1 -- 172.21.15.78:0/3779698690 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb700124ea0 con 0x7fb700129130 2024-09-21T09:20:54.764 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:54.761+0000 7fb6fd7fa640 1 -- 172.21.15.78:0/3779698690 <== mon.1 v2:172.21.15.78:3301/0 5 ==== osd_map(13..13 src has 1..13) v4 ==== 2459+0+0 (secure 0 0 0) 0x7fb6f0014070 con 0x7fb700129130 2024-09-21T09:20:54.766 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:54.765+0000 7fb6fd7fa640 1 -- 172.21.15.78:0/3779698690 <== mon.1 v2:172.21.15.78:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+146310 (secure 0 0 0) 0x7fb700124ea0 con 0x7fb700129130 2024-09-21T09:20:54.901 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:54.900+0000 7fb706b6c640 1 -- 172.21.15.78:0/3779698690 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_command({"prefix": "osd pool create", "pool": "rbd", "pg_num": 8} v 0) v1 -- 0x7fb700124ea0 con 0x7fb700129130 2024-09-21T09:20:55.316 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:55.315+0000 7fb6fd7fa640 1 -- 172.21.15.78:0/3779698690 <== mon.1 v2:172.21.15.78:3301/0 7 ==== mon_command_ack([{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]=0 pool 'rbd' created v14) v1 ==== 109+0+0 (secure 0 0 0) 0x7fb700124ea0 con 0x7fb700129130 2024-09-21T09:20:55.316 INFO:teuthology.orchestra.run.smithi078.stderr:pool 'rbd' created 2024-09-21T09:20:55.319 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:55.317+0000 7fb706b6c640 1 -- 172.21.15.78:0/3779698690 >> v1:172.21.15.78:6816/2602406893 conn(0x7fb6e0047560 legacy=0x7fb6e0049a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:55.319 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:55.318+0000 7fb706b6c640 1 -- 172.21.15.78:0/3779698690 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fb700129130 msgr2=0x7fb700123950 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:55.319 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:55.318+0000 7fb706b6c640 1 --2- 172.21.15.78:0/3779698690 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fb700129130 0x7fb700123950 secure :-1 s=READY pgs=43 cs=0 l=1 rev1=1 crypto rx=0x7fb6f0005fd0 tx=0x7fb6f000c6a0 comp rx=0 tx=0).stop 2024-09-21T09:20:55.319 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:55.318+0000 7fb706b6c640 1 -- 172.21.15.78:0/3779698690 shutdown_connections 2024-09-21T09:20:55.320 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:55.318+0000 7fb706b6c640 1 --2- 172.21.15.78:0/3779698690 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fb70012e5d0 0x7fb700123e90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:55.320 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:55.318+0000 7fb706b6c640 1 --2- 172.21.15.78:0/3779698690 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fb700129130 0x7fb700123950 unknown :-1 s=CLOSED pgs=43 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:55.320 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:55.318+0000 7fb706b6c640 1 --2- 172.21.15.78:0/3779698690 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fb7000710c0 0x7fb700123410 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:55.320 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:55.318+0000 7fb706b6c640 1 -- 172.21.15.78:0/3779698690 >> 172.21.15.78:0/3779698690 conn(0x7fb700097590 msgr2=0x7fb700127b70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:20:55.320 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:55.318+0000 7fb706b6c640 1 -- 172.21.15.78:0/3779698690 shutdown_connections 2024-09-21T09:20:55.320 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:55.318+0000 7fb706b6c640 1 -- 172.21.15.78:0/3779698690 wait complete. 2024-09-21T09:20:55.330 DEBUG:teuthology.orchestra.run.smithi078:> rbd --cluster ceph pool init rbd 2024-09-21T09:20:55.381 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:55.380+0000 7f57b0784c00 1 -- 172.21.15.78:0/1266943232 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x55962e532ea0 msgr2=0x55962e533280 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:55.382 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:55.380+0000 7f57b0784c00 1 --2- 172.21.15.78:0/1266943232 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x55962e532ea0 0x55962e533280 secure :-1 s=READY pgs=44 cs=0 l=1 rev1=1 crypto rx=0x7f57a0004f40 tx=0x7f57a00461d0 comp rx=0 tx=0).stop 2024-09-21T09:20:55.382 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:55.380+0000 7f57b0784c00 1 -- 172.21.15.78:0/1266943232 shutdown_connections 2024-09-21T09:20:55.382 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:55.380+0000 7f57b0784c00 1 --2- 172.21.15.78:0/1266943232 >> v2:172.21.15.78:3300/0 conn(0x55962e5337c0 0x55962e536810 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:55.382 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:55.380+0000 7f57b0784c00 1 --2- 172.21.15.78:0/1266943232 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x55962e532ea0 0x55962e533280 unknown :-1 s=CLOSED pgs=44 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:55.382 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:55.380+0000 7f57b0784c00 1 -- 172.21.15.78:0/1266943232 >> 172.21.15.78:0/1266943232 conn(0x55962e4a5a00 msgr2=0x55962e4a5e10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:20:55.382 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:55.380+0000 7f57b0784c00 1 -- 172.21.15.78:0/1266943232 shutdown_connections 2024-09-21T09:20:55.382 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:55.380+0000 7f57b0784c00 1 -- 172.21.15.78:0/1266943232 wait complete. 2024-09-21T09:20:55.384 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:55.381+0000 7f57b0784c00 1 Processor -- start 2024-09-21T09:20:55.384 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:55.381+0000 7f57b0784c00 1 -- start start 2024-09-21T09:20:55.384 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:55.381+0000 7f57b0784c00 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x55962e4a1e30 0x55962e49f2b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:55.385 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:55.381+0000 7f57b0784c00 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x55962e532ea0 0x55962e49f7f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:55.385 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:55.381+0000 7f57b0784c00 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x55962e5337c0 0x55962e53a040 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:55.385 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:55.381+0000 7f57b0784c00 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 v1 -- 0x55962e37e500 con 0x55962e5337c0 2024-09-21T09:20:55.385 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:55.381+0000 7f57b0784c00 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 v1 -- 0x55962e319b90 con 0x55962e4a1e30 2024-09-21T09:20:55.385 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:55.381+0000 7f57b0784c00 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 v1 -- 0x55962e319830 con 0x55962e532ea0 2024-09-21T09:20:55.385 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:55.381+0000 7f57af281640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x55962e4a1e30 0x55962e49f2b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:55.385 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:55.381+0000 7f57b04e3640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x55962e5337c0 0x55962e53a040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:55.385 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:55.381+0000 7f57af281640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x55962e4a1e30 0x55962e49f2b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3301/0 says I am v2:172.21.15.78:32784/0 (socket says 172.21.15.78:32784) 2024-09-21T09:20:55.385 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:55.381+0000 7f57af281640 1 -- 172.21.15.78:0/2711351461 learned_addr learned my addr 172.21.15.78:0/2711351461 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:20:55.385 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:55.381+0000 7f57a7fff640 1 --2- 172.21.15.78:0/2711351461 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x55962e532ea0 0x55962e49f7f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:55.385 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:55.381+0000 7f57b04e3640 1 -- 172.21.15.78:0/2711351461 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x55962e532ea0 msgr2=0x55962e49f7f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:55.385 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:55.381+0000 7f57b04e3640 1 --2- 172.21.15.78:0/2711351461 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x55962e532ea0 0x55962e49f7f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:55.385 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:55.381+0000 7f57b04e3640 1 -- 172.21.15.78:0/2711351461 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x55962e4a1e30 msgr2=0x55962e49f2b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:55.386 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:55.381+0000 7f57b04e3640 1 --2- 172.21.15.78:0/2711351461 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x55962e4a1e30 0x55962e49f2b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:55.386 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:55.381+0000 7f57b04e3640 1 -- 172.21.15.78:0/2711351461 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f57a800a260 con 0x55962e5337c0 2024-09-21T09:20:55.386 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:55.381+0000 7f57af281640 1 --2- 172.21.15.78:0/2711351461 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x55962e4a1e30 0x55962e49f2b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:20:55.386 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:55.382+0000 7f57b04e3640 1 --2- 172.21.15.78:0/2711351461 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x55962e5337c0 0x55962e53a040 secure :-1 s=READY pgs=52 cs=0 l=1 rev1=1 crypto rx=0x7f57a800d600 tx=0x7f57a800dac0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:20:55.386 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:55.382+0000 7f57a77fe640 1 -- 172.21.15.78:0/2711351461 <== mon.0 v2:172.21.15.78:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f57a800f930 con 0x55962e5337c0 2024-09-21T09:20:55.386 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:55.382+0000 7f57a77fe640 1 -- 172.21.15.78:0/2711351461 <== mon.0 v2:172.21.15.78:3300/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7f57a800fa90 con 0x55962e5337c0 2024-09-21T09:20:55.386 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:55.382+0000 7f57b0784c00 1 -- 172.21.15.78:0/2711351461 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f57a0004ba0 con 0x55962e5337c0 2024-09-21T09:20:55.386 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:55.382+0000 7f57a77fe640 1 -- 172.21.15.78:0/2711351461 <== mon.0 v2:172.21.15.78:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f57a800fea0 con 0x55962e5337c0 2024-09-21T09:20:55.386 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:55.382+0000 7f57b0784c00 1 -- 172.21.15.78:0/2711351461 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55962e328c50 con 0x55962e5337c0 2024-09-21T09:20:55.386 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:55.383+0000 7f57a77fe640 1 -- 172.21.15.78:0/2711351461 <== mon.0 v2:172.21.15.78:3300/0 4 ==== mgrmap(e 5) v1 ==== 55212+0+0 (secure 0 0 0) 0x7f57a8020430 con 0x55962e5337c0 2024-09-21T09:20:55.386 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:55.383+0000 7f57a77fe640 1 -- 172.21.15.78:0/2711351461 <== mon.0 v2:172.21.15.78:3300/0 5 ==== osd_map(14..14 src has 1..14) v4 ==== 2801+0+0 (secure 0 0 0) 0x7f57a805c060 con 0x55962e5337c0 2024-09-21T09:20:55.386 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:55.383+0000 7f57b0784c00 1 -- 172.21.15.78:0/2711351461 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "osd pool application enable","pool": "rbd","app": "rbd"} v 0) v1 -- 0x55962e31c090 con 0x55962e5337c0 2024-09-21T09:20:56.316 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:56.314+0000 7f57a77fe640 1 -- 172.21.15.78:0/2711351461 <== mon.0 v2:172.21.15.78:3300/0 6 ==== mon_command_ack([{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]=0 enabled application 'rbd' on pool 'rbd' v15) v1 ==== 141+0+0 (secure 0 0 0) 0x7f57a8039b80 con 0x55962e5337c0 2024-09-21T09:20:56.316 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:56.314+0000 7f57b0784c00 1 -- 172.21.15.78:0/2711351461 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_get_version(what=osdmap handle=1) v1 -- 0x55962e343990 con 0x55962e5337c0 2024-09-21T09:20:56.319 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:56.318+0000 7f57a77fe640 1 -- 172.21.15.78:0/2711351461 <== mon.0 v2:172.21.15.78:3300/0 7 ==== mon_get_version_reply(handle=1 version=15) v2 ==== 24+0+0 (secure 0 0 0) 0x7f57a805b1f0 con 0x55962e5337c0 2024-09-21T09:20:56.319 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:56.318+0000 7f57ae27f640 1 -- 172.21.15.78:0/2711351461 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({osdmap=15}) v3 -- 0x7f5794002ae0 con 0x55962e5337c0 2024-09-21T09:20:56.322 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:56.321+0000 7f57a77fe640 1 -- 172.21.15.78:0/2711351461 <== mon.0 v2:172.21.15.78:3300/0 8 ==== osd_map(15..15 src has 1..15) v4 ==== 650+0+0 (secure 0 0 0) 0x7f57a803bb50 con 0x55962e5337c0 2024-09-21T09:20:56.323 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:56.322+0000 7f57b0784c00 1 -- 172.21.15.78:0/2711351461 --> v1:172.21.15.78:6802/4276488585 -- osd_op(unknown.0.0:1 2.3 2:c4c92e5a:::rbd_trash:head [create] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e15) v8 -- 0x55962e53b0f0 con 0x55962e54de60 2024-09-21T09:20:56.330 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:56.329+0000 7f57a7fff640 1 -- 172.21.15.78:0/2711351461 <== osd.3 v1:172.21.15.78:6802/4276488585 1 ==== osd_op_reply(1 rbd_trash [create] v15'1 uv1 ondisk = 0) v8 ==== 153+0+0 (unknown 2722885549 0 0) 0x7f579800a040 con 0x55962e54de60 2024-09-21T09:20:56.332 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:56.331+0000 7f57b0784c00 1 -- 172.21.15.78:0/2711351461 --> v1:172.21.15.78:6802/4276488585 -- osd_op(unknown.0.0:2 2.2 2:5cea7035:::rbd_info:head [call rbd.metadata_list in=33b] snapc 0=[] ondisk+read+known_if_redirected+supports_pool_eio e15) v8 -- 0x55962e5a1700 con 0x55962e54de60 2024-09-21T09:20:56.333 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:56.332+0000 7f57a7fff640 1 -- 172.21.15.78:0/2711351461 <== osd.3 v1:172.21.15.78:6802/4276488585 2 ==== osd_op_reply(2 rbd_info [call] v0'0 uv0 ondisk = -2 ((2) No such file or directory)) v8 ==== 152+0+0 (unknown 2354864950 0 0) 0x7f579800a040 con 0x55962e54de60 2024-09-21T09:20:56.333 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:56.332+0000 7f57b0784c00 1 -- 172.21.15.78:0/2711351461 --> v1:172.21.15.78:6802/4276488585 -- osd_op(unknown.0.0:3 2.2 2:5cea7035:::rbd_info:head [read 0~0] snapc 0=[] ondisk+read+known_if_redirected+supports_pool_eio e15) v8 -- 0x55962e5a07f0 con 0x55962e54de60 2024-09-21T09:20:56.333 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:56.332+0000 7f57a7fff640 1 -- 172.21.15.78:0/2711351461 <== osd.3 v1:172.21.15.78:6802/4276488585 3 ==== osd_op_reply(3 rbd_info [read 0~0] v0'0 uv0 ondisk = -2 ((2) No such file or directory)) v8 ==== 152+0+0 (unknown 2366413114 0 0) 0x7f579800a040 con 0x55962e54de60 2024-09-21T09:20:56.333 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:56.332+0000 7f57ae27f640 1 -- 172.21.15.78:0/2711351461 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- pool_op(create unmanaged snap pool 2 tid 4 name v0) v4 -- 0x7f5794003f30 con 0x55962e5337c0 2024-09-21T09:20:57.330 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:57.329+0000 7f57a77fe640 1 -- 172.21.15.78:0/2711351461 <== mon.0 v2:172.21.15.78:3300/0 9 ==== pool_op_reply(tid 4 (0) Success v16) v1 ==== 55+0+0 (secure 0 0 0) 0x7f57a803d370 con 0x55962e5337c0 2024-09-21T09:20:57.330 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:57.329+0000 7f57a77fe640 1 -- 172.21.15.78:0/2711351461 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({osdmap=16}) v3 -- 0x7f579004c940 con 0x55962e5337c0 2024-09-21T09:20:57.331 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:57.329+0000 7f57a77fe640 1 -- 172.21.15.78:0/2711351461 <== mon.0 v2:172.21.15.78:3300/0 10 ==== osd_map(16..16 src has 1..16) v4 ==== 626+0+0 (secure 0 0 0) 0x7f57a803bdb0 con 0x55962e5337c0 2024-09-21T09:20:57.331 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:57.329+0000 7f57ae27f640 1 -- 172.21.15.78:0/2711351461 --> v1:172.21.15.78:6802/4276488585 -- osd_op(unknown.0.0:5 2.2 2:5cea7035:::rbd_info:head [create,write 0~8 in=8b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e16) v8 -- 0x7f5794005830 con 0x55962e54de60 2024-09-21T09:20:57.334 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:57.333+0000 7f57a7fff640 1 -- 172.21.15.78:0/2711351461 <== osd.3 v1:172.21.15.78:6802/4276488585 4 ==== osd_op_reply(5 rbd_info [create,write 0~8] v16'1 uv1 ondisk = 0) v8 ==== 194+0+0 (unknown 1230306805 0 0) 0x7f579800a040 con 0x55962e54de60 2024-09-21T09:20:57.334 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:57.333+0000 7f57ae27f640 1 -- 172.21.15.78:0/2711351461 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- pool_op(delete unmanaged snap pool 2 tid 6 name v16) v4 -- 0x7f5794005340 con 0x55962e5337c0 2024-09-21T09:20:58.334 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.332+0000 7f57a77fe640 1 -- 172.21.15.78:0/2711351461 <== mon.0 v2:172.21.15.78:3300/0 11 ==== pool_op_reply(tid 6 (0) Success v17) v1 ==== 43+0+0 (secure 0 0 0) 0x7f57a803d550 con 0x55962e5337c0 2024-09-21T09:20:58.334 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.332+0000 7f57a77fe640 1 -- 172.21.15.78:0/2711351461 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({osdmap=17}) v3 -- 0x7f579004cad0 con 0x55962e5337c0 2024-09-21T09:20:58.334 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.332+0000 7f57a77fe640 1 -- 172.21.15.78:0/2711351461 <== mon.0 v2:172.21.15.78:3300/0 12 ==== osd_map(17..17 src has 1..17) v4 ==== 654+0+0 (secure 0 0 0) 0x7f57a803a120 con 0x55962e5337c0 2024-09-21T09:20:58.334 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.332+0000 7f57ae27f640 1 -- 172.21.15.78:0/2711351461 --> v1:172.21.15.78:6802/4276488585 -- osd_op(unknown.0.0:7 2.2 2:5cea7035:::rbd_info:head [write 0~19 in=19b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e17) v8 -- 0x7f57940060d0 con 0x55962e54de60 2024-09-21T09:20:58.345 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.343+0000 7f57a7fff640 1 -- 172.21.15.78:0/2711351461 <== osd.3 v1:172.21.15.78:6802/4276488585 5 ==== osd_op_reply(7 rbd_info [write 0~19] v17'2 uv2 ondisk = 0) v8 ==== 152+0+0 (unknown 3144429724 0 0) 0x7f579800a040 con 0x55962e54de60 2024-09-21T09:20:58.345 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.343+0000 7f57b0784c00 1 -- 172.21.15.78:0/2711351461 >> v1:172.21.15.78:6802/4276488585 conn(0x55962e54de60 legacy=0x55962e550250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:58.345 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.343+0000 7f57b0784c00 1 -- 172.21.15.78:0/2711351461 >> v1:172.21.15.78:6816/2602406893 conn(0x7f5790045e50 legacy=0x7f5790048310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:58.345 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.343+0000 7f57b0784c00 1 -- 172.21.15.78:0/2711351461 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x55962e5337c0 msgr2=0x55962e53a040 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:58.345 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.343+0000 7f57b0784c00 1 --2- 172.21.15.78:0/2711351461 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x55962e5337c0 0x55962e53a040 secure :-1 s=READY pgs=52 cs=0 l=1 rev1=1 crypto rx=0x7f57a800d600 tx=0x7f57a800dac0 comp rx=0 tx=0).stop 2024-09-21T09:20:58.345 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.343+0000 7f57b04e3640 1 -- 172.21.15.78:0/2711351461 reap_dead start 2024-09-21T09:20:58.345 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.343+0000 7f57b0784c00 1 -- 172.21.15.78:0/2711351461 shutdown_connections 2024-09-21T09:20:58.345 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.343+0000 7f57b0784c00 1 -- 172.21.15.78:0/2711351461 >> 172.21.15.78:0/2711351461 conn(0x55962e4a5a00 msgr2=0x55962e48de90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:20:58.345 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.343+0000 7f57b0784c00 1 -- 172.21.15.78:0/2711351461 shutdown_connections 2024-09-21T09:20:58.346 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.344+0000 7f57b0784c00 1 -- 172.21.15.78:0/2711351461 wait complete. 2024-09-21T09:20:58.349 INFO:tasks.ceph:Starting mds daemons in cluster ceph... 2024-09-21T09:20:58.349 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph config log 1 --format=json 2024-09-21T09:20:58.350 INFO:tasks.daemonwatchdog.daemon_watchdog:watchdog starting 2024-09-21T09:20:58.455 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.453+0000 7fec80365640 1 -- 172.21.15.78:0/1782920537 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fec7806fd20 msgr2=0x7fec7812d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:58.456 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.453+0000 7fec80365640 1 --2- 172.21.15.78:0/1782920537 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fec7806fd20 0x7fec7812d990 secure :-1 s=READY pgs=45 cs=0 l=1 rev1=1 crypto rx=0x7fec68009890 tx=0x7fec6802a410 comp rx=0 tx=0).stop 2024-09-21T09:20:58.456 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.453+0000 7fec80365640 1 -- 172.21.15.78:0/1782920537 shutdown_connections 2024-09-21T09:20:58.456 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.453+0000 7fec80365640 1 --2- 172.21.15.78:0/1782920537 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fec7806fd20 0x7fec7812d990 unknown :-1 s=CLOSED pgs=45 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:58.456 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.453+0000 7fec80365640 1 --2- 172.21.15.78:0/1782920537 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fec7812ded0 0x7fec781302c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:58.456 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.453+0000 7fec80365640 1 -- 172.21.15.78:0/1782920537 >> 172.21.15.78:0/1782920537 conn(0x7fec78098500 msgr2=0x7fec78098910 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:20:58.456 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.453+0000 7fec80365640 1 -- 172.21.15.78:0/1782920537 shutdown_connections 2024-09-21T09:20:58.456 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.453+0000 7fec80365640 1 -- 172.21.15.78:0/1782920537 wait complete. 2024-09-21T09:20:58.456 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.454+0000 7fec80365640 1 Processor -- start 2024-09-21T09:20:58.456 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.454+0000 7fec80365640 1 -- start start 2024-09-21T09:20:58.458 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.455+0000 7fec80365640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fec7806fd20 0x7fec781c1e90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:58.458 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.455+0000 7fec80365640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fec780952e0 0x7fec781c23d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:58.458 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.455+0000 7fec80365640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fec7812ded0 0x7fec78090e50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:58.458 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.455+0000 7fec80365640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fec780965d0 con 0x7fec7806fd20 2024-09-21T09:20:58.458 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.455+0000 7fec80365640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fec780945b0 con 0x7fec780952e0 2024-09-21T09:20:58.459 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.455+0000 7fec80365640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 v1 -- 0x7fec781c2910 con 0x7fec7812ded0 2024-09-21T09:20:58.459 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.455+0000 7fec7d900640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fec780952e0 0x7fec781c23d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:58.459 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.455+0000 7fec7e101640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fec7806fd20 0x7fec781c1e90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:58.459 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.455+0000 7fec7d900640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fec780952e0 0x7fec781c23d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3301/0 says I am v2:172.21.15.78:32792/0 (socket says 172.21.15.78:32792) 2024-09-21T09:20:58.459 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.455+0000 7fec7d900640 1 -- 172.21.15.78:0/3640497743 learned_addr learned my addr 172.21.15.78:0/3640497743 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:20:58.459 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.455+0000 7fec7e902640 1 --2- 172.21.15.78:0/3640497743 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fec7812ded0 0x7fec78090e50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:58.459 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.455+0000 7fec7d900640 1 -- 172.21.15.78:0/3640497743 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fec7812ded0 msgr2=0x7fec78090e50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:58.459 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.455+0000 7fec7d900640 1 --2- 172.21.15.78:0/3640497743 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fec7812ded0 0x7fec78090e50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:58.459 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.455+0000 7fec7d900640 1 -- 172.21.15.78:0/3640497743 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fec7806fd20 msgr2=0x7fec781c1e90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:58.459 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.455+0000 7fec7d900640 1 --2- 172.21.15.78:0/3640497743 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fec7806fd20 0x7fec781c1e90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:58.459 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.455+0000 7fec7d900640 1 -- 172.21.15.78:0/3640497743 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fec70009490 con 0x7fec780952e0 2024-09-21T09:20:58.459 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.455+0000 7fec7e101640 1 --2- 172.21.15.78:0/3640497743 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fec7806fd20 0x7fec781c1e90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:20:58.459 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.455+0000 7fec7d900640 1 --2- 172.21.15.78:0/3640497743 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fec780952e0 0x7fec781c23d0 secure :-1 s=READY pgs=46 cs=0 l=1 rev1=1 crypto rx=0x7fec6802a930 tx=0x7fec68003c10 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:20:58.460 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.455+0000 7fec6effd640 1 -- 172.21.15.78:0/3640497743 <== mon.1 v2:172.21.15.78:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fec68005220 con 0x7fec780952e0 2024-09-21T09:20:58.460 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.455+0000 7fec6effd640 1 -- 172.21.15.78:0/3640497743 <== mon.1 v2:172.21.15.78:3301/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7fec68005490 con 0x7fec780952e0 2024-09-21T09:20:58.460 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.455+0000 7fec6effd640 1 -- 172.21.15.78:0/3640497743 <== mon.1 v2:172.21.15.78:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fec68004a00 con 0x7fec780952e0 2024-09-21T09:20:58.460 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.455+0000 7fec80365640 1 -- 172.21.15.78:0/3640497743 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fec68009490 con 0x7fec780952e0 2024-09-21T09:20:58.460 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.455+0000 7fec80365640 1 -- 172.21.15.78:0/3640497743 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fec78093ae0 con 0x7fec780952e0 2024-09-21T09:20:58.460 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.456+0000 7fec6effd640 1 -- 172.21.15.78:0/3640497743 <== mon.1 v2:172.21.15.78:3301/0 4 ==== mgrmap(e 5) v1 ==== 55212+0+0 (secure 0 0 0) 0x7fec68004b60 con 0x7fec780952e0 2024-09-21T09:20:58.460 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.456+0000 7fec80365640 1 -- 172.21.15.78:0/3640497743 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fec780956c0 con 0x7fec780952e0 2024-09-21T09:20:58.460 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.457+0000 7fec6effd640 1 -- 172.21.15.78:0/3640497743 <== mon.1 v2:172.21.15.78:3301/0 5 ==== osd_map(17..17 src has 1..17) v4 ==== 2868+0+0 (secure 0 0 0) 0x7fec6807da80 con 0x7fec780952e0 2024-09-21T09:20:58.461 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.460+0000 7fec6effd640 1 -- 172.21.15.78:0/3640497743 <== mon.1 v2:172.21.15.78:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+146310 (secure 0 0 0) 0x7fec6805f1c0 con 0x7fec780952e0 2024-09-21T09:20:58.598 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.597+0000 7fec80365640 1 -- 172.21.15.78:0/3640497743 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_command({"prefix": "config log", "num": 1, "format": "json"} v 0) v1 -- 0x7fec78093d00 con 0x7fec780952e0 2024-09-21T09:20:58.599 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.597+0000 7fec6effd640 1 -- 172.21.15.78:0/3640497743 <== mon.1 v2:172.21.15.78:3301/0 7 ==== mon_command_ack([{"prefix": "config log", "num": 1, "format": "json"}]=0 v23) v1 ==== 86+0+160 (secure 0 0 0) 0x7fec6805ed80 con 0x7fec780952e0 2024-09-21T09:20:58.599 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:20:58.600 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.599+0000 7fec80365640 1 -- 172.21.15.78:0/3640497743 >> v1:172.21.15.78:6816/2602406893 conn(0x7fec5c047560 legacy=0x7fec5c049a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:58.600 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.599+0000 7fec80365640 1 -- 172.21.15.78:0/3640497743 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fec780952e0 msgr2=0x7fec781c23d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:58.600 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.599+0000 7fec80365640 1 --2- 172.21.15.78:0/3640497743 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fec780952e0 0x7fec781c23d0 secure :-1 s=READY pgs=46 cs=0 l=1 rev1=1 crypto rx=0x7fec6802a930 tx=0x7fec68003c10 comp rx=0 tx=0).stop 2024-09-21T09:20:58.601 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.599+0000 7fec80365640 1 -- 172.21.15.78:0/3640497743 shutdown_connections 2024-09-21T09:20:58.601 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.599+0000 7fec80365640 1 --2- 172.21.15.78:0/3640497743 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fec7812ded0 0x7fec78090e50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:58.601 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.599+0000 7fec80365640 1 --2- 172.21.15.78:0/3640497743 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fec780952e0 0x7fec781c23d0 unknown :-1 s=CLOSED pgs=46 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:58.601 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.599+0000 7fec80365640 1 --2- 172.21.15.78:0/3640497743 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fec7806fd20 0x7fec781c1e90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:58.601 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.599+0000 7fec80365640 1 -- 172.21.15.78:0/3640497743 >> 172.21.15.78:0/3640497743 conn(0x7fec78098500 msgr2=0x7fec7806c470 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:20:58.601 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.599+0000 7fec80365640 1 -- 172.21.15.78:0/3640497743 shutdown_connections 2024-09-21T09:20:58.601 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.599+0000 7fec80365640 1 -- 172.21.15.78:0/3640497743 wait complete. 2024-09-21T09:20:58.610 INFO:teuthology.orchestra.run.smithi078.stdout:[{"version":23,"timestamp":"2024-09-21T09:20:48.332763+0000","name":"","changes":[{"name":"osd.2/osd_mclock_max_capacity_iops_ssd","new_value":"5238.641466"}]}] 2024-09-21T09:20:58.610 INFO:tasks.ceph_manager:config epoch is 23 2024-09-21T09:20:58.611 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-21T09:20:58.611 INFO:tasks.ceph.ceph_manager.ceph:waiting for mgr available 2024-09-21T09:20:58.611 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph mgr dump --format=json 2024-09-21T09:20:58.710 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.708+0000 7f1426ff4640 1 -- 172.21.15.78:0/3336702899 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f142006e710 msgr2=0x7f142006eba0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:58.710 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.708+0000 7f1426ff4640 1 --2- 172.21.15.78:0/3336702899 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f142006e710 0x7f142006eba0 secure :-1 s=READY pgs=29 cs=0 l=1 rev1=1 crypto rx=0x7f1408009860 tx=0x7f140802a410 comp rx=0 tx=0).stop 2024-09-21T09:20:58.710 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.708+0000 7f1426ff4640 1 -- 172.21.15.78:0/3336702899 shutdown_connections 2024-09-21T09:20:58.710 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.708+0000 7f1426ff4640 1 --2- 172.21.15.78:0/3336702899 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f142006e710 0x7f142006eba0 unknown :-1 s=CLOSED pgs=29 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:58.710 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.708+0000 7f1426ff4640 1 --2- 172.21.15.78:0/3336702899 >> v2:172.21.15.78:3300/0 conn(0x7f1420095c00 0x7f142006e1d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:58.710 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.708+0000 7f1426ff4640 1 --2- 172.21.15.78:0/3336702899 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f14200952e0 0x7f14200956c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:58.710 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.708+0000 7f1426ff4640 1 -- 172.21.15.78:0/3336702899 >> 172.21.15.78:0/3336702899 conn(0x7f1420097590 msgr2=0x7f14200979a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:20:58.710 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.708+0000 7f1426ff4640 1 -- 172.21.15.78:0/3336702899 shutdown_connections 2024-09-21T09:20:58.710 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.708+0000 7f1426ff4640 1 -- 172.21.15.78:0/3336702899 wait complete. 2024-09-21T09:20:58.711 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.709+0000 7f1426ff4640 1 Processor -- start 2024-09-21T09:20:58.711 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.709+0000 7f1426ff4640 1 -- start start 2024-09-21T09:20:58.713 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.709+0000 7f1426ff4640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f142006e710 0x7f142012e220 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:58.713 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.709+0000 7f1426ff4640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f14200952e0 0x7f142012e760 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:58.713 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.709+0000 7f1426ff4640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f1420095c00 0x7f142012eca0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:58.713 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.709+0000 7f1426ff4640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1420120330 con 0x7f142006e710 2024-09-21T09:20:58.713 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.709+0000 7f1426ff4640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f14200965d0 con 0x7f14200952e0 2024-09-21T09:20:58.713 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.709+0000 7f1426ff4640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 v1 -- 0x7f1420094500 con 0x7f1420095c00 2024-09-21T09:20:58.713 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.709+0000 7f141ffff640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f14200952e0 0x7f142012e760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:58.713 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.709+0000 7f1424d90640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f142006e710 0x7f142012e220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:58.713 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.709+0000 7f1425591640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f1420095c00 0x7f142012eca0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:58.713 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.709+0000 7f141ffff640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f14200952e0 0x7f142012e760 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3301/0 says I am v2:172.21.15.78:32822/0 (socket says 172.21.15.78:32822) 2024-09-21T09:20:58.713 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.709+0000 7f141ffff640 1 -- 172.21.15.78:0/97871847 learned_addr learned my addr 172.21.15.78:0/97871847 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:20:58.713 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.710+0000 7f1424d90640 1 -- 172.21.15.78:0/97871847 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f1420095c00 msgr2=0x7f142012eca0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:58.713 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.710+0000 7f1424d90640 1 --2- 172.21.15.78:0/97871847 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f1420095c00 0x7f142012eca0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:58.713 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.710+0000 7f1424d90640 1 -- 172.21.15.78:0/97871847 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f14200952e0 msgr2=0x7f142012e760 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:58.714 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.710+0000 7f1424d90640 1 --2- 172.21.15.78:0/97871847 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f14200952e0 0x7f142012e760 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:58.714 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.710+0000 7f1424d90640 1 -- 172.21.15.78:0/97871847 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1418009560 con 0x7f142006e710 2024-09-21T09:20:58.714 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.710+0000 7f1425591640 1 --2- 172.21.15.78:0/97871847 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f1420095c00 0x7f142012eca0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:20:58.714 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.710+0000 7f1424d90640 1 --2- 172.21.15.78:0/97871847 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f142006e710 0x7f142012e220 secure :-1 s=READY pgs=54 cs=0 l=1 rev1=1 crypto rx=0x7f1418005fd0 tx=0x7f141800ee50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:20:58.714 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.710+0000 7f141d7fa640 1 -- 172.21.15.78:0/97871847 <== mon.0 v2:172.21.15.78:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f141800cd60 con 0x7f142006e710 2024-09-21T09:20:58.714 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.710+0000 7f141d7fa640 1 -- 172.21.15.78:0/97871847 <== mon.0 v2:172.21.15.78:3300/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7f1418004020 con 0x7f142006e710 2024-09-21T09:20:58.714 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.710+0000 7f141d7fa640 1 -- 172.21.15.78:0/97871847 <== mon.0 v2:172.21.15.78:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1418004410 con 0x7f142006e710 2024-09-21T09:20:58.714 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.710+0000 7f1426ff4640 1 -- 172.21.15.78:0/97871847 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1408009490 con 0x7f142006e710 2024-09-21T09:20:58.714 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.710+0000 7f1426ff4640 1 -- 172.21.15.78:0/97871847 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1420092a30 con 0x7f142006e710 2024-09-21T09:20:58.714 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.711+0000 7f1426ff4640 1 -- 172.21.15.78:0/97871847 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f142012f9a0 con 0x7f142006e710 2024-09-21T09:20:58.714 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.711+0000 7f141d7fa640 1 -- 172.21.15.78:0/97871847 <== mon.0 v2:172.21.15.78:3300/0 4 ==== mgrmap(e 5) v1 ==== 55212+0+0 (secure 0 0 0) 0x7f14180035f0 con 0x7f142006e710 2024-09-21T09:20:58.714 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.712+0000 7f141d7fa640 1 -- 172.21.15.78:0/97871847 <== mon.0 v2:172.21.15.78:3300/0 5 ==== osd_map(17..17 src has 1..17) v4 ==== 2868+0+0 (secure 0 0 0) 0x7f1418014070 con 0x7f142006e710 2024-09-21T09:20:58.716 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.715+0000 7f141d7fa640 1 -- 172.21.15.78:0/97871847 <== mon.0 v2:172.21.15.78:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+146310 (secure 0 0 0) 0x7f141803ae00 con 0x7f142006e710 2024-09-21T09:20:58.880 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.878+0000 7f1426ff4640 1 -- 172.21.15.78:0/97871847 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "mgr dump", "format": "json"} v 0) v1 -- 0x7f142006eba0 con 0x7f142006e710 2024-09-21T09:20:58.882 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.880+0000 7f141d7fa640 1 -- 172.21.15.78:0/97871847 <== mon.0 v2:172.21.15.78:3300/0 7 ==== mon_command_ack([{"prefix": "mgr dump", "format": "json"}]=0 v5) v1 ==== 74+0+107688 (secure 0 0 0) 0x7f141803e420 con 0x7f142006e710 2024-09-21T09:20:58.883 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:20:58.884 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.883+0000 7f1426ff4640 1 -- 172.21.15.78:0/97871847 >> v1:172.21.15.78:6816/2602406893 conn(0x7f1400047560 legacy=0x7f1400049a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:58.885 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.883+0000 7f1426ff4640 1 -- 172.21.15.78:0/97871847 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f142006e710 msgr2=0x7f142012e220 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:58.885 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.883+0000 7f1426ff4640 1 --2- 172.21.15.78:0/97871847 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f142006e710 0x7f142012e220 secure :-1 s=READY pgs=54 cs=0 l=1 rev1=1 crypto rx=0x7f1418005fd0 tx=0x7f141800ee50 comp rx=0 tx=0).stop 2024-09-21T09:20:58.885 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.883+0000 7f1426ff4640 1 -- 172.21.15.78:0/97871847 shutdown_connections 2024-09-21T09:20:58.885 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.883+0000 7f1426ff4640 1 --2- 172.21.15.78:0/97871847 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f1420095c00 0x7f142012eca0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:58.885 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.883+0000 7f1426ff4640 1 --2- 172.21.15.78:0/97871847 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f14200952e0 0x7f142012e760 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:58.885 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.883+0000 7f1426ff4640 1 --2- 172.21.15.78:0/97871847 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f142006e710 0x7f142012e220 unknown :-1 s=CLOSED pgs=54 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:58.885 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.883+0000 7f1426ff4640 1 -- 172.21.15.78:0/97871847 >> 172.21.15.78:0/97871847 conn(0x7f1420097590 msgr2=0x7f142008b7c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:20:58.885 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.883+0000 7f1426ff4640 1 -- 172.21.15.78:0/97871847 shutdown_connections 2024-09-21T09:20:58.885 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.883+0000 7f1426ff4640 1 -- 172.21.15.78:0/97871847 wait complete. 2024-09-21T09:20:58.894 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":5,"flags":0,"active_gid":4106,"active_name":"x","active_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6816","nonce":2602406893}]},"active_addr":"172.21.15.78:6816/2602406893","active_change":"2024-09-21T09:20:41.135199+0000","active_mgr_features":4540138322906710015,"available":true,"standbys":[{"gid":4102,"name":"y","mgr_features":4540138322906710015,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["iostat","nfs","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":0,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.78:0","nonce":2453110290}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.78:0","nonce":2894754949}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.78:0","nonce":2078474014}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.78:0","nonce":1812672186}]}]} 2024-09-21T09:20:58.897 INFO:tasks.ceph.ceph_manager.ceph:mgr available! 2024-09-21T09:20:58.897 INFO:tasks.ceph.ceph_manager.ceph:waiting for all up 2024-09-21T09:20:58.897 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-09-21T09:20:58.996 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.994+0000 7f404c7e2640 1 -- 172.21.15.78:0/4136093922 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f404412c650 msgr2=0x7f404412ca30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:58.996 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.994+0000 7f404c7e2640 1 --2- 172.21.15.78:0/4136093922 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f404412c650 0x7f404412ca30 secure :-1 s=READY pgs=30 cs=0 l=1 rev1=1 crypto rx=0x7f403c009860 tx=0x7f403c02a3d0 comp rx=0 tx=0).stop 2024-09-21T09:20:58.996 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.994+0000 7f404c7e2640 1 -- 172.21.15.78:0/4136093922 shutdown_connections 2024-09-21T09:20:58.996 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.994+0000 7f404c7e2640 1 --2- 172.21.15.78:0/4136093922 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f404406ff10 0x7f404406e5c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:58.996 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.994+0000 7f404c7e2640 1 --2- 172.21.15.78:0/4136093922 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f404412c650 0x7f404412ca30 unknown :-1 s=CLOSED pgs=30 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:58.996 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.994+0000 7f404c7e2640 1 -- 172.21.15.78:0/4136093922 >> 172.21.15.78:0/4136093922 conn(0x7f404409c980 msgr2=0x7f404409cd90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:20:58.997 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.994+0000 7f404c7e2640 1 -- 172.21.15.78:0/4136093922 shutdown_connections 2024-09-21T09:20:58.997 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.994+0000 7f404c7e2640 1 -- 172.21.15.78:0/4136093922 wait complete. 2024-09-21T09:20:58.997 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.995+0000 7f404c7e2640 1 Processor -- start 2024-09-21T09:20:58.997 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.995+0000 7f404c7e2640 1 -- start start 2024-09-21T09:20:58.999 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.995+0000 7f404c7e2640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f404406eb00 0x7f4044130130 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:58.999 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.995+0000 7f404c7e2640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f404406ff10 0x7f4044130670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:58.999 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.995+0000 7f404c7e2640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f404412c650 0x7f4044130bb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:58.999 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.995+0000 7f404c7e2640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4044133b90 con 0x7f404406eb00 2024-09-21T09:20:58.999 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.995+0000 7f404c7e2640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f40441324f0 con 0x7f404412c650 2024-09-21T09:20:58.999 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.995+0000 7f404c7e2640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 v1 -- 0x7f40441338d0 con 0x7f404406ff10 2024-09-21T09:20:58.999 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.995+0000 7f404a57e640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f404406eb00 0x7f4044130130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:58.999 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.995+0000 7f4049d7d640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f404406ff10 0x7f4044130670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:59.000 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.995+0000 7f404ad7f640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f404412c650 0x7f4044130bb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:59.000 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.995+0000 7f404a57e640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f404406eb00 0x7f4044130130 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3300/0 says I am v2:172.21.15.78:37498/0 (socket says 172.21.15.78:37498) 2024-09-21T09:20:59.000 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.995+0000 7f404a57e640 1 -- 172.21.15.78:0/1013705648 learned_addr learned my addr 172.21.15.78:0/1013705648 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:20:59.000 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.995+0000 7f4049d7d640 1 -- 172.21.15.78:0/1013705648 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f404412c650 msgr2=0x7f4044130bb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:59.000 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.995+0000 7f4049d7d640 1 --2- 172.21.15.78:0/1013705648 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f404412c650 0x7f4044130bb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:59.000 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.995+0000 7f4049d7d640 1 -- 172.21.15.78:0/1013705648 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f404406eb00 msgr2=0x7f4044130130 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:59.000 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.995+0000 7f4049d7d640 1 --2- 172.21.15.78:0/1013705648 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f404406eb00 0x7f4044130130 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:59.000 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.995+0000 7f4049d7d640 1 -- 172.21.15.78:0/1013705648 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4038009900 con 0x7f404406ff10 2024-09-21T09:20:59.000 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.995+0000 7f404ad7f640 1 --2- 172.21.15.78:0/1013705648 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f404412c650 0x7f4044130bb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:20:59.000 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.996+0000 7f4049d7d640 1 --2- 172.21.15.78:0/1013705648 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f404406ff10 0x7f4044130670 secure :-1 s=READY pgs=31 cs=0 l=1 rev1=1 crypto rx=0x7f404406c030 tx=0x7f403800c800 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:20:59.000 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.996+0000 7f4042ffd640 1 -- 172.21.15.78:0/1013705648 <== mon.2 v2:172.21.15.78:3302/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4038003b80 con 0x7f404406ff10 2024-09-21T09:20:59.001 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.996+0000 7f4042ffd640 1 -- 172.21.15.78:0/1013705648 <== mon.2 v2:172.21.15.78:3302/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7f4038003dd0 con 0x7f404406ff10 2024-09-21T09:20:59.001 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.996+0000 7f404c7e2640 1 -- 172.21.15.78:0/1013705648 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f403c009490 con 0x7f404406ff10 2024-09-21T09:20:59.001 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.996+0000 7f4042ffd640 1 -- 172.21.15.78:0/1013705648 <== mon.2 v2:172.21.15.78:3302/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4038004290 con 0x7f404406ff10 2024-09-21T09:20:59.001 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.996+0000 7f404c7e2640 1 -- 172.21.15.78:0/1013705648 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f40441345a0 con 0x7f404406ff10 2024-09-21T09:20:59.001 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.997+0000 7f4042ffd640 1 -- 172.21.15.78:0/1013705648 <== mon.2 v2:172.21.15.78:3302/0 4 ==== mgrmap(e 5) v1 ==== 55212+0+0 (secure 0 0 0) 0x7f403800a040 con 0x7f404406ff10 2024-09-21T09:20:59.001 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.997+0000 7f404c7e2640 1 -- 172.21.15.78:0/1013705648 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f404406d320 con 0x7f404406ff10 2024-09-21T09:20:59.001 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:58.997+0000 7f4042ffd640 1 -- 172.21.15.78:0/1013705648 <== mon.2 v2:172.21.15.78:3302/0 5 ==== osd_map(17..17 src has 1..17) v4 ==== 2868+0+0 (secure 0 0 0) 0x7f403805cf00 con 0x7f404406ff10 2024-09-21T09:20:59.002 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.001+0000 7f4042ffd640 1 -- 172.21.15.78:0/1013705648 <== mon.2 v2:172.21.15.78:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+146310 (secure 0 0 0) 0x7f403803ac10 con 0x7f404406ff10 2024-09-21T09:20:59.137 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.136+0000 7f404c7e2640 1 -- 172.21.15.78:0/1013705648 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f404406e5c0 con 0x7f404406ff10 2024-09-21T09:20:59.137 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.136+0000 7f4042ffd640 1 -- 172.21.15.78:0/1013705648 <== mon.2 v2:172.21.15.78:3302/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v17) v1 ==== 74+0+8799 (secure 0 0 0) 0x7f403801b070 con 0x7f404406ff10 2024-09-21T09:20:59.138 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:20:59.138 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":17,"fsid":"3633ad52-1063-4406-97d9-d53c339f43ca","created":"2024-09-21T09:20:38.248415+0000","modified":"2024-09-21T09:20:58.322673+0000","last_up_change":"2024-09-21T09:20:49.276020+0000","last_in_change":"2024-09-21T09:20:40.330925+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T09:20:51.169034+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-21T09:20:54.918007+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":17,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"6b18d222-0705-4495-a01c-3e2ba2ab4cff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6808","nonce":1449157914}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6809","nonce":1449157914}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6811","nonce":1449157914}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6810","nonce":1449157914}]},"public_addr":"172.21.15.78:6808/1449157914","cluster_addr":"172.21.15.78:6809/1449157914","heartbeat_back_addr":"172.21.15.78:6811/1449157914","heartbeat_front_addr":"172.21.15.78:6810/1449157914","state":["exists","up"]},{"osd":1,"uuid":"2d15b0fa-66c2-4bda-b64b-d72cbbb16fb1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6800","nonce":3190796132}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6801","nonce":3190796132}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6805","nonce":3190796132}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6803","nonce":3190796132}]},"public_addr":"172.21.15.78:6800/3190796132","cluster_addr":"172.21.15.78:6801/3190796132","heartbeat_back_addr":"172.21.15.78:6805/3190796132","heartbeat_front_addr":"172.21.15.78:6803/3190796132","state":["exists","up"]},{"osd":2,"uuid":"0bf75d8f-017d-488d-8a2e-c3fa3b2a29c2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6812","nonce":3442117044}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6813","nonce":3442117044}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6815","nonce":3442117044}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6814","nonce":3442117044}]},"public_addr":"172.21.15.78:6812/3442117044","cluster_addr":"172.21.15.78:6813/3442117044","heartbeat_back_addr":"172.21.15.78:6815/3442117044","heartbeat_front_addr":"172.21.15.78:6814/3442117044","state":["exists","up"]},{"osd":3,"uuid":"5e3788c8-b89a-4614-b3ec-7ee1250ddbe0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6802","nonce":4276488585}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6804","nonce":4276488585}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6807","nonce":4276488585}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6806","nonce":4276488585}]},"public_addr":"172.21.15.78:6802/4276488585","cluster_addr":"172.21.15.78:6804/4276488585","heartbeat_back_addr":"172.21.15.78:6807/4276488585","heartbeat_front_addr":"172.21.15.78:6806/4276488585","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:20:45.499319+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:20:45.489695+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:20:45.541959+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:20:45.476915+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T09:20:59.140 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.138+0000 7f404c7e2640 1 -- 172.21.15.78:0/1013705648 >> v1:172.21.15.78:6816/2602406893 conn(0x7f4030047560 legacy=0x7f4030049a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:59.140 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.138+0000 7f404c7e2640 1 -- 172.21.15.78:0/1013705648 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f404406ff10 msgr2=0x7f4044130670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:59.140 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.138+0000 7f404c7e2640 1 --2- 172.21.15.78:0/1013705648 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f404406ff10 0x7f4044130670 secure :-1 s=READY pgs=31 cs=0 l=1 rev1=1 crypto rx=0x7f404406c030 tx=0x7f403800c800 comp rx=0 tx=0).stop 2024-09-21T09:20:59.140 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.138+0000 7f404c7e2640 1 -- 172.21.15.78:0/1013705648 shutdown_connections 2024-09-21T09:20:59.140 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.138+0000 7f404c7e2640 1 --2- 172.21.15.78:0/1013705648 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f404412c650 0x7f4044130bb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:59.140 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.138+0000 7f404c7e2640 1 --2- 172.21.15.78:0/1013705648 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f404406ff10 0x7f4044130670 unknown :-1 s=CLOSED pgs=31 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:59.140 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.138+0000 7f404c7e2640 1 --2- 172.21.15.78:0/1013705648 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f404406eb00 0x7f4044130130 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:59.140 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.138+0000 7f404c7e2640 1 -- 172.21.15.78:0/1013705648 >> 172.21.15.78:0/1013705648 conn(0x7f404409c980 msgr2=0x7f4044125b50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:20:59.140 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.138+0000 7f404c7e2640 1 -- 172.21.15.78:0/1013705648 shutdown_connections 2024-09-21T09:20:59.140 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.138+0000 7f404c7e2640 1 -- 172.21.15.78:0/1013705648 wait complete. 2024-09-21T09:20:59.150 INFO:tasks.ceph.ceph_manager.ceph:all up! 2024-09-21T09:20:59.150 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-09-21T09:20:59.250 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.248+0000 7fdfd4983640 1 -- 172.21.15.78:0/3702275478 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fdfd0098c10 msgr2=0x7fdfd012ef90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:59.250 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.248+0000 7fdfd4983640 1 --2- 172.21.15.78:0/3702275478 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fdfd0098c10 0x7fdfd012ef90 secure :-1 s=READY pgs=57 cs=0 l=1 rev1=1 crypto rx=0x7fdfc4009830 tx=0x7fdfc402a3f0 comp rx=0 tx=0).stop 2024-09-21T09:20:59.250 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.248+0000 7fdfd4983640 1 -- 172.21.15.78:0/3702275478 shutdown_connections 2024-09-21T09:20:59.250 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.248+0000 7fdfd4983640 1 --2- 172.21.15.78:0/3702275478 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fdfd012f4d0 0x7fdfd01318c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:59.250 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.248+0000 7fdfd4983640 1 --2- 172.21.15.78:0/3702275478 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fdfd0098c10 0x7fdfd012ef90 unknown :-1 s=CLOSED pgs=57 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:59.250 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.248+0000 7fdfd4983640 1 -- 172.21.15.78:0/3702275478 >> 172.21.15.78:0/3702275478 conn(0x7fdfd009abe0 msgr2=0x7fdfd0070ef0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:20:59.251 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.248+0000 7fdfd4983640 1 -- 172.21.15.78:0/3702275478 shutdown_connections 2024-09-21T09:20:59.251 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.248+0000 7fdfd4983640 1 -- 172.21.15.78:0/3702275478 wait complete. 2024-09-21T09:20:59.251 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.249+0000 7fdfd4983640 1 Processor -- start 2024-09-21T09:20:59.251 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.249+0000 7fdfd4983640 1 -- start start 2024-09-21T09:20:59.253 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.249+0000 7fdfd4983640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fdfd00963d0 0x7fdfd012d160 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:59.253 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.249+0000 7fdfd4983640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fdfd0098c10 0x7fdfd012d6a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:59.253 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.249+0000 7fdfd4983640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fdfd012f4d0 0x7fdfd012dbe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:59.253 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.249+0000 7fdfd4983640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdfd012c210 con 0x7fdfd012f4d0 2024-09-21T09:20:59.253 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.249+0000 7fdfd4983640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fdfd012c860 con 0x7fdfd00963d0 2024-09-21T09:20:59.253 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.249+0000 7fdfd4983640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 v1 -- 0x7fdfd012cc30 con 0x7fdfd0098c10 2024-09-21T09:20:59.253 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.249+0000 7fdfce59c640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fdfd00963d0 0x7fdfd012d160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:59.253 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.249+0000 7fdfcdd9b640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fdfd0098c10 0x7fdfd012d6a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:59.253 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.249+0000 7fdfce59c640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fdfd00963d0 0x7fdfd012d160 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3301/0 says I am v2:172.21.15.78:32868/0 (socket says 172.21.15.78:32868) 2024-09-21T09:20:59.253 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.249+0000 7fdfce59c640 1 -- 172.21.15.78:0/2452737043 learned_addr learned my addr 172.21.15.78:0/2452737043 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:20:59.253 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.249+0000 7fdfced9d640 1 --2- 172.21.15.78:0/2452737043 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fdfd012f4d0 0x7fdfd012dbe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:59.253 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.250+0000 7fdfcdd9b640 1 -- 172.21.15.78:0/2452737043 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fdfd00963d0 msgr2=0x7fdfd012d160 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:59.254 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.250+0000 7fdfcdd9b640 1 --2- 172.21.15.78:0/2452737043 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fdfd00963d0 0x7fdfd012d160 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:59.254 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.250+0000 7fdfcdd9b640 1 -- 172.21.15.78:0/2452737043 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fdfd012f4d0 msgr2=0x7fdfd012dbe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:59.254 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.250+0000 7fdfcdd9b640 1 --2- 172.21.15.78:0/2452737043 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fdfd012f4d0 0x7fdfd012dbe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:59.254 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.250+0000 7fdfcdd9b640 1 -- 172.21.15.78:0/2452737043 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fdfc8009840 con 0x7fdfd0098c10 2024-09-21T09:20:59.254 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.250+0000 7fdfced9d640 1 --2- 172.21.15.78:0/2452737043 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fdfd012f4d0 0x7fdfd012dbe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:20:59.254 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.250+0000 7fdfcdd9b640 1 --2- 172.21.15.78:0/2452737043 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fdfd0098c10 0x7fdfd012d6a0 secure :-1 s=READY pgs=32 cs=0 l=1 rev1=1 crypto rx=0x7fdfd009c470 tx=0x7fdfc4003c10 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:20:59.254 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.250+0000 7fdfbeffd640 1 -- 172.21.15.78:0/2452737043 <== mon.2 v2:172.21.15.78:3302/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdfc4005c90 con 0x7fdfd0098c10 2024-09-21T09:20:59.254 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.250+0000 7fdfbeffd640 1 -- 172.21.15.78:0/2452737043 <== mon.2 v2:172.21.15.78:3302/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7fdfc4005110 con 0x7fdfd0098c10 2024-09-21T09:20:59.254 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.250+0000 7fdfd4983640 1 -- 172.21.15.78:0/2452737043 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fdfc4009490 con 0x7fdfd0098c10 2024-09-21T09:20:59.254 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.250+0000 7fdfbeffd640 1 -- 172.21.15.78:0/2452737043 <== mon.2 v2:172.21.15.78:3302/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdfc40054e0 con 0x7fdfd0098c10 2024-09-21T09:20:59.254 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.250+0000 7fdfd4983640 1 -- 172.21.15.78:0/2452737043 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fdfd01293c0 con 0x7fdfd0098c10 2024-09-21T09:20:59.254 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.251+0000 7fdfbeffd640 1 -- 172.21.15.78:0/2452737043 <== mon.2 v2:172.21.15.78:3302/0 4 ==== mgrmap(e 5) v1 ==== 55212+0+0 (secure 0 0 0) 0x7fdfc40047a0 con 0x7fdfd0098c10 2024-09-21T09:20:59.254 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.251+0000 7fdfd4983640 1 -- 172.21.15.78:0/2452737043 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fdfd012a7f0 con 0x7fdfd0098c10 2024-09-21T09:20:59.254 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.251+0000 7fdfbeffd640 1 -- 172.21.15.78:0/2452737043 <== mon.2 v2:172.21.15.78:3302/0 5 ==== osd_map(17..17 src has 1..17) v4 ==== 2868+0+0 (secure 0 0 0) 0x7fdfc407dba0 con 0x7fdfd0098c10 2024-09-21T09:20:59.256 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.255+0000 7fdfbeffd640 1 -- 172.21.15.78:0/2452737043 <== mon.2 v2:172.21.15.78:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+146310 (secure 0 0 0) 0x7fdfc407de60 con 0x7fdfd0098c10 2024-09-21T09:20:59.394 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.392+0000 7fdfd4983640 1 -- 172.21.15.78:0/2452737043 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fdfd00967b0 con 0x7fdfd0098c10 2024-09-21T09:20:59.394 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.393+0000 7fdfbeffd640 1 -- 172.21.15.78:0/2452737043 <== mon.2 v2:172.21.15.78:3302/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v17) v1 ==== 74+0+8799 (secure 0 0 0) 0x7fdfd012a7f0 con 0x7fdfd0098c10 2024-09-21T09:20:59.394 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:20:59.395 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":17,"fsid":"3633ad52-1063-4406-97d9-d53c339f43ca","created":"2024-09-21T09:20:38.248415+0000","modified":"2024-09-21T09:20:58.322673+0000","last_up_change":"2024-09-21T09:20:49.276020+0000","last_in_change":"2024-09-21T09:20:40.330925+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T09:20:51.169034+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-21T09:20:54.918007+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":17,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"6b18d222-0705-4495-a01c-3e2ba2ab4cff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6808","nonce":1449157914}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6809","nonce":1449157914}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6811","nonce":1449157914}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6810","nonce":1449157914}]},"public_addr":"172.21.15.78:6808/1449157914","cluster_addr":"172.21.15.78:6809/1449157914","heartbeat_back_addr":"172.21.15.78:6811/1449157914","heartbeat_front_addr":"172.21.15.78:6810/1449157914","state":["exists","up"]},{"osd":1,"uuid":"2d15b0fa-66c2-4bda-b64b-d72cbbb16fb1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6800","nonce":3190796132}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6801","nonce":3190796132}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6805","nonce":3190796132}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6803","nonce":3190796132}]},"public_addr":"172.21.15.78:6800/3190796132","cluster_addr":"172.21.15.78:6801/3190796132","heartbeat_back_addr":"172.21.15.78:6805/3190796132","heartbeat_front_addr":"172.21.15.78:6803/3190796132","state":["exists","up"]},{"osd":2,"uuid":"0bf75d8f-017d-488d-8a2e-c3fa3b2a29c2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6812","nonce":3442117044}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6813","nonce":3442117044}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6815","nonce":3442117044}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6814","nonce":3442117044}]},"public_addr":"172.21.15.78:6812/3442117044","cluster_addr":"172.21.15.78:6813/3442117044","heartbeat_back_addr":"172.21.15.78:6815/3442117044","heartbeat_front_addr":"172.21.15.78:6814/3442117044","state":["exists","up"]},{"osd":3,"uuid":"5e3788c8-b89a-4614-b3ec-7ee1250ddbe0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6802","nonce":4276488585}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6804","nonce":4276488585}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6807","nonce":4276488585}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6806","nonce":4276488585}]},"public_addr":"172.21.15.78:6802/4276488585","cluster_addr":"172.21.15.78:6804/4276488585","heartbeat_back_addr":"172.21.15.78:6807/4276488585","heartbeat_front_addr":"172.21.15.78:6806/4276488585","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:20:45.499319+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:20:45.489695+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:20:45.541959+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:20:45.476915+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T09:20:59.396 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.395+0000 7fdfd4983640 1 -- 172.21.15.78:0/2452737043 >> v1:172.21.15.78:6816/2602406893 conn(0x7fdfb0047560 legacy=0x7fdfb0049a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:59.396 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.395+0000 7fdfd4983640 1 -- 172.21.15.78:0/2452737043 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fdfd0098c10 msgr2=0x7fdfd012d6a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:59.396 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.395+0000 7fdfd4983640 1 --2- 172.21.15.78:0/2452737043 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fdfd0098c10 0x7fdfd012d6a0 secure :-1 s=READY pgs=32 cs=0 l=1 rev1=1 crypto rx=0x7fdfd009c470 tx=0x7fdfc4003c10 comp rx=0 tx=0).stop 2024-09-21T09:20:59.397 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.395+0000 7fdfd4983640 1 -- 172.21.15.78:0/2452737043 shutdown_connections 2024-09-21T09:20:59.397 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.395+0000 7fdfd4983640 1 --2- 172.21.15.78:0/2452737043 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fdfd012f4d0 0x7fdfd012dbe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:59.397 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.395+0000 7fdfd4983640 1 --2- 172.21.15.78:0/2452737043 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fdfd0098c10 0x7fdfd012d6a0 unknown :-1 s=CLOSED pgs=32 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:59.397 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.395+0000 7fdfd4983640 1 --2- 172.21.15.78:0/2452737043 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fdfd00963d0 0x7fdfd012d160 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:59.397 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.395+0000 7fdfd4983640 1 -- 172.21.15.78:0/2452737043 >> 172.21.15.78:0/2452737043 conn(0x7fdfd009abe0 msgr2=0x7fdfd0070920 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:20:59.397 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.395+0000 7fdfd4983640 1 -- 172.21.15.78:0/2452737043 shutdown_connections 2024-09-21T09:20:59.397 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.395+0000 7fdfd4983640 1 -- 172.21.15.78:0/2452737043 wait complete. 2024-09-21T09:20:59.407 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.0 flush_pg_stats 2024-09-21T09:20:59.407 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.1 flush_pg_stats 2024-09-21T09:20:59.408 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.2 flush_pg_stats 2024-09-21T09:20:59.408 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.3 flush_pg_stats 2024-09-21T09:20:59.514 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.513+0000 7f05a5b33640 1 -- 172.21.15.78:0/3775534383 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f05a0128d70 msgr2=0x7f05a0129150 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:59.514 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.513+0000 7f05a5b33640 1 --2- 172.21.15.78:0/3775534383 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f05a0128d70 0x7f05a0129150 secure :-1 s=READY pgs=60 cs=0 l=1 rev1=1 crypto rx=0x7f0590013f30 tx=0x7f0590032960 comp rx=0 tx=0).stop 2024-09-21T09:20:59.514 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.513+0000 7f05a5b33640 1 -- 172.21.15.78:0/3775534383 shutdown_connections 2024-09-21T09:20:59.515 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.513+0000 7f05a5b33640 1 --2- 172.21.15.78:0/3775534383 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f05a0070b80 0x7f05a012bdf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:59.515 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.513+0000 7f05a5b33640 1 --2- 172.21.15.78:0/3775534383 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f05a012c330 0x7f05a012c7c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:59.515 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.513+0000 7f05a5b33640 1 --2- 172.21.15.78:0/3775534383 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f05a0128d70 0x7f05a0129150 unknown :-1 s=CLOSED pgs=60 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:59.515 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.513+0000 7f05a5b33640 1 -- 172.21.15.78:0/3775534383 >> 172.21.15.78:0/3775534383 conn(0x7f05a009c8a0 msgr2=0x7f05a009ccb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:20:59.515 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.513+0000 7f05a5b33640 1 -- 172.21.15.78:0/3775534383 shutdown_connections 2024-09-21T09:20:59.515 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.513+0000 7faf263ec640 1 -- 172.21.15.78:0/3976907106 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7faf200952e0 msgr2=0x7faf200956c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:59.515 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.513+0000 7faf263ec640 1 --2- 172.21.15.78:0/3976907106 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7faf200952e0 0x7faf200956c0 secure :-1 s=READY pgs=61 cs=0 l=1 rev1=1 crypto rx=0x7faf10009930 tx=0x7faf1002a510 comp rx=0 tx=0).stop 2024-09-21T09:20:59.516 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.513+0000 7fab4943a640 1 -- 172.21.15.78:0/2471548342 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fab44071ab0 msgr2=0x7fab44071f40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:59.516 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.513+0000 7fab4943a640 1 --2- 172.21.15.78:0/2471548342 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fab44071ab0 0x7fab44071f40 secure :-1 s=READY pgs=62 cs=0 l=1 rev1=1 crypto rx=0x7fab3c009c10 tx=0x7fab3c02a730 comp rx=0 tx=0).stop 2024-09-21T09:20:59.516 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.513+0000 7fab4943a640 1 -- 172.21.15.78:0/2471548342 shutdown_connections 2024-09-21T09:20:59.516 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.513+0000 7fab4943a640 1 --2- 172.21.15.78:0/2471548342 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fab44097550 0x7fab44071570 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:59.516 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.513+0000 7fab4943a640 1 --2- 172.21.15.78:0/2471548342 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fab44071ab0 0x7fab44071f40 unknown :-1 s=CLOSED pgs=62 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:59.516 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.513+0000 7fab4943a640 1 --2- 172.21.15.78:0/2471548342 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fab44096c30 0x7fab44097010 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:59.516 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.513+0000 7fab4943a640 1 -- 172.21.15.78:0/2471548342 >> 172.21.15.78:0/2471548342 conn(0x7fab4409ed50 msgr2=0x7fab4409b220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:20:59.517 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.513+0000 7f05a5b33640 1 -- 172.21.15.78:0/3775534383 wait complete. 2024-09-21T09:20:59.517 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.513+0000 7faf263ec640 1 -- 172.21.15.78:0/3976907106 shutdown_connections 2024-09-21T09:20:59.517 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.513+0000 7faf263ec640 1 --2- 172.21.15.78:0/3976907106 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7faf2006f420 0x7faf2006f890 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:59.517 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.513+0000 7faf263ec640 1 --2- 172.21.15.78:0/3976907106 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7faf200952e0 0x7faf200956c0 unknown :-1 s=CLOSED pgs=61 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:59.517 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.513+0000 7faf263ec640 1 -- 172.21.15.78:0/3976907106 >> 172.21.15.78:0/3976907106 conn(0x7faf20098500 msgr2=0x7faf20098910 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:20:59.518 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.513+0000 7fab4943a640 1 -- 172.21.15.78:0/2471548342 shutdown_connections 2024-09-21T09:20:59.518 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.513+0000 7fab4943a640 1 -- 172.21.15.78:0/2471548342 wait complete. 2024-09-21T09:20:59.518 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.513+0000 7faf263ec640 1 -- 172.21.15.78:0/3976907106 shutdown_connections 2024-09-21T09:20:59.518 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.513+0000 7faf263ec640 1 -- 172.21.15.78:0/3976907106 wait complete. 2024-09-21T09:20:59.519 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.513+0000 7f94119b5640 1 -- 172.21.15.78:0/3858251963 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f940c097130 msgr2=0x7f940c12f300 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:59.519 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.513+0000 7f94119b5640 1 --2- 172.21.15.78:0/3858251963 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f940c097130 0x7f940c12f300 secure :-1 s=READY pgs=33 cs=0 l=1 rev1=1 crypto rx=0x7f940000fb60 tx=0x7f940002e600 comp rx=0 tx=0).stop 2024-09-21T09:20:59.519 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.513+0000 7f94119b5640 1 -- 172.21.15.78:0/3858251963 shutdown_connections 2024-09-21T09:20:59.519 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.513+0000 7f94119b5640 1 --2- 172.21.15.78:0/3858251963 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f940c06f820 0x7f940c131850 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:59.519 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.513+0000 7f94119b5640 1 --2- 172.21.15.78:0/3858251963 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f940c097130 0x7f940c12f300 unknown :-1 s=CLOSED pgs=33 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:59.519 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.513+0000 7f94119b5640 1 -- 172.21.15.78:0/3858251963 >> 172.21.15.78:0/3858251963 conn(0x7f940c097c00 msgr2=0x7f940c098010 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:20:59.519 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.514+0000 7f94119b5640 1 -- 172.21.15.78:0/3858251963 shutdown_connections 2024-09-21T09:20:59.520 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.514+0000 7f94119b5640 1 -- 172.21.15.78:0/3858251963 wait complete. 2024-09-21T09:20:59.520 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.514+0000 7f05a5b33640 1 Processor -- start 2024-09-21T09:20:59.520 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.514+0000 7f05a5b33640 1 -- start start 2024-09-21T09:20:59.520 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.514+0000 7faf263ec640 1 Processor -- start 2024-09-21T09:20:59.521 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.514+0000 7fab4943a640 1 Processor -- start 2024-09-21T09:20:59.521 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.514+0000 7f05a5b33640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f05a0070b80 0x7f05a0114ff0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:59.521 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.514+0000 7f05a5b33640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f05a0128d70 0x7f05a0115530 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:59.521 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.514+0000 7f05a5b33640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f05a012c330 0x7f05a0115a70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:59.521 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.514+0000 7f05a5b33640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f05a0117340 con 0x7f05a0128d70 2024-09-21T09:20:59.521 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.514+0000 7f05a5b33640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f05a0117b20 con 0x7f05a012c330 2024-09-21T09:20:59.521 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.514+0000 7f05a5b33640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 v1 -- 0x7f05a0117ef0 con 0x7f05a0070b80 2024-09-21T09:20:59.522 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.514+0000 7fab4943a640 1 -- start start 2024-09-21T09:20:59.522 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.515+0000 7faf263ec640 1 -- start start 2024-09-21T09:20:59.522 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.515+0000 7f94119b5640 1 Processor -- start 2024-09-21T09:20:59.522 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.515+0000 7f059f7fe640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f05a0070b80 0x7f05a0114ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:59.523 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.515+0000 7f059effd640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f05a0128d70 0x7f05a0115530 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:59.523 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.515+0000 7f059ffff640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f05a012c330 0x7f05a0115a70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:59.523 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.515+0000 7f059f7fe640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f05a0070b80 0x7f05a0114ff0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3302/0 says I am v2:172.21.15.78:35820/0 (socket says 172.21.15.78:35820) 2024-09-21T09:20:59.523 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.515+0000 7f059f7fe640 1 -- 172.21.15.78:0/1553556118 learned_addr learned my addr 172.21.15.78:0/1553556118 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:20:59.523 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.515+0000 7f059effd640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f05a0128d70 0x7f05a0115530 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3300/0 says I am v2:172.21.15.78:37548/0 (socket says 172.21.15.78:37548) 2024-09-21T09:20:59.523 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.515+0000 7f94119b5640 1 -- start start 2024-09-21T09:20:59.523 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.515+0000 7fab4943a640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fab44071ab0 0x7fab44132510 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:59.523 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.515+0000 7fab4943a640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fab44096c30 0x7fab44132a50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:59.524 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.515+0000 7f059f7fe640 1 -- 172.21.15.78:0/1553556118 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f05a012c330 msgr2=0x7f05a0115a70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:59.524 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.515+0000 7f059f7fe640 1 --2- 172.21.15.78:0/1553556118 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f05a012c330 0x7f05a0115a70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:59.524 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.515+0000 7f059f7fe640 1 -- 172.21.15.78:0/1553556118 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f05a0128d70 msgr2=0x7f05a0115530 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:59.524 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.515+0000 7f059f7fe640 1 --2- 172.21.15.78:0/1553556118 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f05a0128d70 0x7f05a0115530 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:59.524 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.515+0000 7f059f7fe640 1 -- 172.21.15.78:0/1553556118 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0590013b90 con 0x7f05a0070b80 2024-09-21T09:20:59.524 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.515+0000 7faf263ec640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7faf2006f420 0x7faf201bfcc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:59.524 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.515+0000 7fab4943a640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fab44097550 0x7fab44132f90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:59.525 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.515+0000 7faf263ec640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7faf200952e0 0x7faf201c0200 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:59.525 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.515+0000 7faf263ec640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7faf20095c00 0x7faf20091cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:59.525 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.515+0000 7fab4943a640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fab4406caa0 con 0x7fab44071ab0 2024-09-21T09:20:59.525 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.515+0000 7fab4943a640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fab4406d360 con 0x7fab44097550 2024-09-21T09:20:59.525 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.515+0000 7fab4943a640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 v1 -- 0x7fab4406dd80 con 0x7fab44096c30 2024-09-21T09:20:59.525 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.515+0000 7faf263ec640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 v1 -- 0x7faf20094d10 con 0x7faf200952e0 2024-09-21T09:20:59.525 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.515+0000 7faf263ec640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 v1 -- 0x7faf200940f0 con 0x7faf2006f420 2024-09-21T09:20:59.525 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.515+0000 7faf263ec640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 v1 -- 0x7faf200965d0 con 0x7faf20095c00 2024-09-21T09:20:59.526 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.515+0000 7fab42ffd640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fab44071ab0 0x7fab44132510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:59.526 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.515+0000 7fab42ffd640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fab44071ab0 0x7fab44132510 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3300/0 says I am v2:172.21.15.78:37564/0 (socket says 172.21.15.78:37564) 2024-09-21T09:20:59.526 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.515+0000 7fab42ffd640 1 -- 172.21.15.78:0/3793099369 learned_addr learned my addr 172.21.15.78:0/3793099369 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:20:59.526 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.515+0000 7faf1ffff640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7faf2006f420 0x7faf201bfcc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:59.526 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.515+0000 7faf1ffff640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7faf2006f420 0x7faf201bfcc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3301/0 says I am v2:172.21.15.78:32924/0 (socket says 172.21.15.78:32924) 2024-09-21T09:20:59.526 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.515+0000 7faf1ffff640 1 -- 172.21.15.78:0/1937097236 learned_addr learned my addr 172.21.15.78:0/1937097236 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:20:59.527 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.515+0000 7fab427fc640 1 --2- 172.21.15.78:0/3793099369 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fab44096c30 0x7fab44132a50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:59.527 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.515+0000 7fab437fe640 1 --2- 172.21.15.78:0/3793099369 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fab44097550 0x7fab44132f90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:59.527 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.515+0000 7f94119b5640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f940c06ed30 0x7f940c1ba310 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:59.527 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.515+0000 7f94119b5640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f940c06f820 0x7f940c1ba850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:59.527 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.515+0000 7f94119b5640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f940c097130 0x7f940c1bad90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:59.527 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.516+0000 7faf1f7fe640 1 --2- 172.21.15.78:0/1937097236 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7faf200952e0 0x7faf201c0200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:59.527 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.516+0000 7faf24989640 1 --2- 172.21.15.78:0/1937097236 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7faf20095c00 0x7faf20091cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:59.528 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.516+0000 7f94119b5640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f940c090be0 con 0x7f940c097130 2024-09-21T09:20:59.528 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.516+0000 7faf1ffff640 1 -- 172.21.15.78:0/1937097236 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7faf20095c00 msgr2=0x7faf20091cc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:59.528 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.516+0000 7faf1ffff640 1 --2- 172.21.15.78:0/1937097236 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7faf20095c00 0x7faf20091cc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:59.528 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.516+0000 7faf1ffff640 1 -- 172.21.15.78:0/1937097236 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7faf200952e0 msgr2=0x7faf201c0200 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:59.528 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.516+0000 7faf1ffff640 1 --2- 172.21.15.78:0/1937097236 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7faf200952e0 0x7faf201c0200 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:59.528 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.516+0000 7faf1ffff640 1 -- 172.21.15.78:0/1937097236 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7faf10009560 con 0x7faf2006f420 2024-09-21T09:20:59.529 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.516+0000 7f94119b5640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f940c090230 con 0x7f940c06f820 2024-09-21T09:20:59.529 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.516+0000 7f940a7fc640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f940c06f820 0x7f940c1ba850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:59.529 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.516+0000 7f940a7fc640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f940c06f820 0x7f940c1ba850 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3301/0 says I am v2:172.21.15.78:32952/0 (socket says 172.21.15.78:32952) 2024-09-21T09:20:59.529 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.516+0000 7f940a7fc640 1 -- 172.21.15.78:0/3317798998 learned_addr learned my addr 172.21.15.78:0/3317798998 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:20:59.530 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.516+0000 7f94119b5640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 v1 -- 0x7f940c08f000 con 0x7f940c06ed30 2024-09-21T09:20:59.530 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.516+0000 7f940b7fe640 1 --2- 172.21.15.78:0/3317798998 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f940c097130 0x7f940c1bad90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:59.530 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.516+0000 7faf1ffff640 1 --2- 172.21.15.78:0/1937097236 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7faf2006f420 0x7faf201bfcc0 secure :-1 s=READY pgs=47 cs=0 l=1 rev1=1 crypto rx=0x7faf10009230 tx=0x7faf10003960 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:20:59.530 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.516+0000 7f059f7fe640 1 --2- 172.21.15.78:0/1553556118 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f05a0070b80 0x7f05a0114ff0 secure :-1 s=READY pgs=34 cs=0 l=1 rev1=1 crypto rx=0x7f0590000c00 tx=0x7f05900042f0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:20:59.531 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.516+0000 7f940affd640 1 --2- 172.21.15.78:0/3317798998 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f940c06ed30 0x7f940c1ba310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:59.531 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.516+0000 7faf1cff9640 1 -- 172.21.15.78:0/1937097236 <== mon.1 v2:172.21.15.78:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7faf1003c470 con 0x7faf2006f420 2024-09-21T09:20:59.531 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.516+0000 7fab42ffd640 1 -- 172.21.15.78:0/3793099369 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fab44096c30 msgr2=0x7fab44132a50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:59.531 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.516+0000 7fab42ffd640 1 --2- 172.21.15.78:0/3793099369 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fab44096c30 0x7fab44132a50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:59.531 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.516+0000 7fab42ffd640 1 -- 172.21.15.78:0/3793099369 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fab44097550 msgr2=0x7fab44132f90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:59.531 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.516+0000 7fab42ffd640 1 --2- 172.21.15.78:0/3793099369 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fab44097550 0x7fab44132f90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:59.531 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.516+0000 7fab42ffd640 1 -- 172.21.15.78:0/3793099369 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fab3c009840 con 0x7fab44071ab0 2024-09-21T09:20:59.532 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.516+0000 7fab42ffd640 1 --2- 172.21.15.78:0/3793099369 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fab44071ab0 0x7fab44132510 secure :-1 s=READY pgs=63 cs=0 l=1 rev1=1 crypto rx=0x7fab3400d950 tx=0x7fab3400de10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:20:59.532 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.516+0000 7faf263ec640 1 -- 172.21.15.78:0/1937097236 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7faf2011fb30 con 0x7faf2006f420 2024-09-21T09:20:59.532 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.516+0000 7faf1cff9640 1 -- 172.21.15.78:0/1937097236 <== mon.1 v2:172.21.15.78:3301/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7faf1003c6c0 con 0x7faf2006f420 2024-09-21T09:20:59.532 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.516+0000 7faf1cff9640 1 -- 172.21.15.78:0/1937097236 <== mon.1 v2:172.21.15.78:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7faf1003ca50 con 0x7faf2006f420 2024-09-21T09:20:59.532 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.516+0000 7faf263ec640 1 -- 172.21.15.78:0/1937097236 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7faf201c0740 con 0x7faf2006f420 2024-09-21T09:20:59.533 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.516+0000 7fab2b7fe640 1 -- 172.21.15.78:0/3793099369 <== mon.0 v2:172.21.15.78:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fab34009a30 con 0x7fab44071ab0 2024-09-21T09:20:59.533 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.517+0000 7f0597fff640 1 -- 172.21.15.78:0/1553556118 <== mon.2 v2:172.21.15.78:3302/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0590005220 con 0x7f05a0070b80 2024-09-21T09:20:59.533 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.517+0000 7f0597fff640 1 -- 172.21.15.78:0/1553556118 <== mon.2 v2:172.21.15.78:3302/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7f0590005380 con 0x7f05a0070b80 2024-09-21T09:20:59.533 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.517+0000 7f0597fff640 1 -- 172.21.15.78:0/1553556118 <== mon.2 v2:172.21.15.78:3302/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f05900044e0 con 0x7f05a0070b80 2024-09-21T09:20:59.533 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.516+0000 7fab4943a640 1 -- 172.21.15.78:0/3793099369 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fab440712d0 con 0x7fab44071ab0 2024-09-21T09:20:59.534 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.516+0000 7fab4943a640 1 -- 172.21.15.78:0/3793099369 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fab44128850 con 0x7fab44071ab0 2024-09-21T09:20:59.534 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.517+0000 7f940a7fc640 1 -- 172.21.15.78:0/3317798998 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f940c06ed30 msgr2=0x7f940c1ba310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:59.534 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.517+0000 7f940a7fc640 1 --2- 172.21.15.78:0/3317798998 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f940c06ed30 0x7f940c1ba310 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:59.534 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.517+0000 7f940a7fc640 1 -- 172.21.15.78:0/3317798998 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f940c097130 msgr2=0x7f940c1bad90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:59.535 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.517+0000 7f940a7fc640 1 --2- 172.21.15.78:0/3317798998 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f940c097130 0x7f940c1bad90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:59.535 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.517+0000 7f940a7fc640 1 -- 172.21.15.78:0/3317798998 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f940000f790 con 0x7f940c06f820 2024-09-21T09:20:59.535 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.517+0000 7f940a7fc640 1 --2- 172.21.15.78:0/3317798998 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f940c06f820 0x7f940c1ba850 secure :-1 s=READY pgs=48 cs=0 l=1 rev1=1 crypto rx=0x7f940000fc80 tx=0x7f9400003b90 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:20:59.535 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.517+0000 7f05a5b33640 1 -- 172.21.15.78:0/1553556118 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f05a0116ab0 con 0x7f05a0070b80 2024-09-21T09:20:59.535 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.517+0000 7f05a5b33640 1 -- 172.21.15.78:0/1553556118 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f05a008c890 con 0x7f05a0070b80 2024-09-21T09:20:59.535 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.518+0000 7fab4943a640 1 -- 172.21.15.78:0/3793099369 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_get_version(what=osdmap handle=1) v1 -- 0x7fab44097010 con 0x7fab44071ab0 2024-09-21T09:20:59.536 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.518+0000 7faf263ec640 1 -- 172.21.15.78:0/1937097236 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_get_version(what=osdmap handle=1) v1 -- 0x7faf201309b0 con 0x7faf2006f420 2024-09-21T09:20:59.536 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.517+0000 7f940affd640 1 --2- 172.21.15.78:0/3317798998 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f940c06ed30 0x7f940c1ba310 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:20:59.536 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.518+0000 7f93f3fff640 1 -- 172.21.15.78:0/3317798998 <== mon.1 v2:172.21.15.78:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9400005220 con 0x7f940c06f820 2024-09-21T09:20:59.536 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.517+0000 7f940b7fe640 1 --2- 172.21.15.78:0/3317798998 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f940c097130 0x7f940c1bad90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:20:59.536 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.518+0000 7f93f3fff640 1 -- 172.21.15.78:0/3317798998 <== mon.1 v2:172.21.15.78:3301/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7f9400005470 con 0x7f940c06f820 2024-09-21T09:20:59.536 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.518+0000 7faf1cff9640 1 -- 172.21.15.78:0/1937097236 <== mon.1 v2:172.21.15.78:3301/0 4 ==== mgrmap(e 5) v1 ==== 55212+0+0 (secure 0 0 0) 0x7faf1003cbb0 con 0x7faf2006f420 2024-09-21T09:20:59.537 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.518+0000 7fab2b7fe640 1 -- 172.21.15.78:0/3793099369 <== mon.0 v2:172.21.15.78:3300/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7fab34009c80 con 0x7fab44071ab0 2024-09-21T09:20:59.537 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.518+0000 7fab2b7fe640 1 -- 172.21.15.78:0/3793099369 <== mon.0 v2:172.21.15.78:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fab34004280 con 0x7fab44071ab0 2024-09-21T09:20:59.537 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.518+0000 7f05a5b33640 1 -- 172.21.15.78:0/1553556118 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_get_version(what=osdmap handle=1) v1 -- 0x7f05a008cb70 con 0x7f05a0070b80 2024-09-21T09:20:59.537 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.518+0000 7faf1cff9640 1 -- 172.21.15.78:0/1937097236 <== mon.1 v2:172.21.15.78:3301/0 5 ==== osd_map(17..17 src has 1..17) v4 ==== 2868+0+0 (secure 0 0 0) 0x7faf1007df00 con 0x7faf2006f420 2024-09-21T09:20:59.538 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.518+0000 7faf1cff9640 1 -- 172.21.15.78:0/1937097236 --> v1:172.21.15.78:6812/3442117044 -- command(tid 1: {"prefix": "get_command_descriptions"}) v1 -- 0x7faf0c051910 con 0x7faf0c04dc20 2024-09-21T09:20:59.538 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.518+0000 7faf1cff9640 1 -- 172.21.15.78:0/1937097236 <== mon.1 v2:172.21.15.78:3301/0 6 ==== mon_get_version_reply(handle=1 version=17) v2 ==== 24+0+0 (secure 0 0 0) 0x7faf10083490 con 0x7faf2006f420 2024-09-21T09:20:59.538 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.518+0000 7f93f3fff640 1 -- 172.21.15.78:0/3317798998 <== mon.1 v2:172.21.15.78:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9400004a00 con 0x7f940c06f820 2024-09-21T09:20:59.538 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.518+0000 7f94119b5640 1 -- 172.21.15.78:0/3317798998 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f940c08ff50 con 0x7f940c06f820 2024-09-21T09:20:59.538 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.518+0000 7f94119b5640 1 -- 172.21.15.78:0/3317798998 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f940c1bd810 con 0x7f940c06f820 2024-09-21T09:20:59.538 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.519+0000 7fab2b7fe640 1 -- 172.21.15.78:0/3793099369 <== mon.0 v2:172.21.15.78:3300/0 4 ==== mgrmap(e 5) v1 ==== 55212+0+0 (secure 0 0 0) 0x7fab340035f0 con 0x7fab44071ab0 2024-09-21T09:20:59.539 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.519+0000 7f0597fff640 1 -- 172.21.15.78:0/1553556118 <== mon.2 v2:172.21.15.78:3302/0 4 ==== mgrmap(e 5) v1 ==== 55212+0+0 (secure 0 0 0) 0x7f0590003800 con 0x7f05a0070b80 2024-09-21T09:20:59.539 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.519+0000 7fab2b7fe640 1 -- 172.21.15.78:0/3793099369 <== mon.0 v2:172.21.15.78:3300/0 5 ==== osd_map(17..17 src has 1..17) v4 ==== 2868+0+0 (secure 0 0 0) 0x7fab3405d540 con 0x7fab44071ab0 2024-09-21T09:20:59.539 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.519+0000 7f0597fff640 1 -- 172.21.15.78:0/1553556118 <== mon.2 v2:172.21.15.78:3302/0 5 ==== osd_map(17..17 src has 1..17) v4 ==== 2868+0+0 (secure 0 0 0) 0x7f0590085db0 con 0x7f05a0070b80 2024-09-21T09:20:59.540 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.519+0000 7f0597fff640 1 -- 172.21.15.78:0/1553556118 --> v1:172.21.15.78:6800/3190796132 -- command(tid 1: {"prefix": "get_command_descriptions"}) v1 -- 0x7f05800519b0 con 0x7f058004dcc0 2024-09-21T09:20:59.540 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.520+0000 7f0597fff640 1 -- 172.21.15.78:0/1553556118 <== mon.2 v2:172.21.15.78:3302/0 6 ==== mon_get_version_reply(handle=1 version=17) v2 ==== 24+0+0 (secure 0 0 0) 0x7f05900883d0 con 0x7f05a0070b80 2024-09-21T09:20:59.540 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.520+0000 7f94119b5640 1 -- 172.21.15.78:0/3317798998 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_get_version(what=osdmap handle=1) v1 -- 0x7f940c06f110 con 0x7f940c06f820 2024-09-21T09:20:59.540 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.520+0000 7fab2b7fe640 1 -- 172.21.15.78:0/3793099369 --> v1:172.21.15.78:6808/1449157914 -- command(tid 1: {"prefix": "get_command_descriptions"}) v1 -- 0x7fab300519b0 con 0x7fab3004dcc0 2024-09-21T09:20:59.540 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.520+0000 7f93f3fff640 1 -- 172.21.15.78:0/3317798998 <== mon.1 v2:172.21.15.78:3301/0 4 ==== mgrmap(e 5) v1 ==== 55212+0+0 (secure 0 0 0) 0x7f9400004b60 con 0x7f940c06f820 2024-09-21T09:20:59.541 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.520+0000 7fab2b7fe640 1 -- 172.21.15.78:0/3793099369 <== mon.0 v2:172.21.15.78:3300/0 6 ==== mon_get_version_reply(handle=1 version=17) v2 ==== 24+0+0 (secure 0 0 0) 0x7fab3405d8f0 con 0x7fab44071ab0 2024-09-21T09:20:59.541 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.520+0000 7faf1cff9640 1 -- 172.21.15.78:0/1937097236 <== osd.2 v1:172.21.15.78:6812/3442117044 1 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+26118 (unknown 0 0 1686565167) 0x7faf0c051910 con 0x7faf0c04dc20 2024-09-21T09:20:59.541 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.520+0000 7f93f3fff640 1 -- 172.21.15.78:0/3317798998 <== mon.1 v2:172.21.15.78:3301/0 5 ==== osd_map(17..17 src has 1..17) v4 ==== 2868+0+0 (secure 0 0 0) 0x7f9400081a90 con 0x7f940c06f820 2024-09-21T09:20:59.541 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.520+0000 7f93f3fff640 1 -- 172.21.15.78:0/3317798998 --> v1:172.21.15.78:6802/4276488585 -- command(tid 1: {"prefix": "get_command_descriptions"}) v1 -- 0x7f93f8051930 con 0x7f93f804dc40 2024-09-21T09:20:59.541 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.520+0000 7f93f3fff640 1 -- 172.21.15.78:0/3317798998 <== mon.1 v2:172.21.15.78:3301/0 6 ==== mon_get_version_reply(handle=1 version=17) v2 ==== 24+0+0 (secure 0 0 0) 0x7f94000862c0 con 0x7f940c06f820 2024-09-21T09:20:59.542 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.521+0000 7f0597fff640 1 -- 172.21.15.78:0/1553556118 <== osd.1 v1:172.21.15.78:6800/3190796132 1 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+26118 (unknown 0 0 1686565167) 0x7f05800519b0 con 0x7f058004dcc0 2024-09-21T09:20:59.542 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.521+0000 7fab2b7fe640 1 -- 172.21.15.78:0/3793099369 <== osd.0 v1:172.21.15.78:6808/1449157914 1 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+26118 (unknown 0 0 1686565167) 0x7fab300519b0 con 0x7fab3004dcc0 2024-09-21T09:20:59.542 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.522+0000 7f93f3fff640 1 -- 172.21.15.78:0/3317798998 <== osd.3 v1:172.21.15.78:6802/4276488585 1 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+26118 (unknown 0 0 1686565167) 0x7f93f8051930 con 0x7f93f804dc40 2024-09-21T09:20:59.543 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.533+0000 7faf263ec640 1 -- 172.21.15.78:0/1937097236 --> v1:172.21.15.78:6812/3442117044 -- command(tid 2: {"prefix": "flush_pg_stats"}) v1 -- 0x7faf2004e3a0 con 0x7faf0c04dc20 2024-09-21T09:20:59.543 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.534+0000 7faf1cff9640 1 -- 172.21.15.78:0/1937097236 <== osd.2 v1:172.21.15.78:6812/3442117044 2 ==== command_reply(tid 2: 0 ) v1 ==== 8+0+11 (unknown 0 0 1129848789) 0x7faf2004e3a0 con 0x7faf0c04dc20 2024-09-21T09:20:59.544 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.534+0000 7faf263ec640 1 -- 172.21.15.78:0/1937097236 >> v1:172.21.15.78:6812/3442117044 conn(0x7faf0c04dc20 legacy=0x7faf0c0500a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:59.544 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.534+0000 7faf263ec640 1 -- 172.21.15.78:0/1937097236 >> v1:172.21.15.78:6816/2602406893 conn(0x7faf0c047590 legacy=0x7faf0c049a50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:59.544 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.534+0000 7faf263ec640 1 -- 172.21.15.78:0/1937097236 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7faf2006f420 msgr2=0x7faf201bfcc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:59.544 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.534+0000 7faf263ec640 1 --2- 172.21.15.78:0/1937097236 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7faf2006f420 0x7faf201bfcc0 secure :-1 s=READY pgs=47 cs=0 l=1 rev1=1 crypto rx=0x7faf10009230 tx=0x7faf10003960 comp rx=0 tx=0).stop 2024-09-21T09:20:59.544 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.534+0000 7faf24989640 1 -- 172.21.15.78:0/1937097236 reap_dead start 2024-09-21T09:20:59.544 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.534+0000 7faf263ec640 1 -- 172.21.15.78:0/1937097236 shutdown_connections 2024-09-21T09:20:59.544 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.534+0000 7faf263ec640 1 -- 172.21.15.78:0/1937097236 >> 172.21.15.78:0/1937097236 conn(0x7faf20098500 msgr2=0x7faf2008afc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:20:59.544 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.534+0000 7faf263ec640 1 -- 172.21.15.78:0/1937097236 shutdown_connections 2024-09-21T09:20:59.544 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.534+0000 7faf263ec640 1 -- 172.21.15.78:0/1937097236 wait complete. 2024-09-21T09:20:59.545 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.534+0000 7f05a5b33640 1 -- 172.21.15.78:0/1553556118 --> v1:172.21.15.78:6800/3190796132 -- command(tid 2: {"prefix": "flush_pg_stats"}) v1 -- 0x7f05a0089990 con 0x7f058004dcc0 2024-09-21T09:20:59.545 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.534+0000 7fab4943a640 1 -- 172.21.15.78:0/3793099369 --> v1:172.21.15.78:6808/1449157914 -- command(tid 2: {"prefix": "flush_pg_stats"}) v1 -- 0x7fab44133a80 con 0x7fab3004dcc0 2024-09-21T09:20:59.545 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.535+0000 7f94119b5640 1 -- 172.21.15.78:0/3317798998 --> v1:172.21.15.78:6802/4276488585 -- command(tid 2: {"prefix": "flush_pg_stats"}) v1 -- 0x7f940c06d8e0 con 0x7f93f804dc40 2024-09-21T09:20:59.545 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.535+0000 7f0597fff640 1 -- 172.21.15.78:0/1553556118 <== osd.1 v1:172.21.15.78:6800/3190796132 2 ==== command_reply(tid 2: 0 ) v1 ==== 8+0+11 (unknown 0 0 3659784045) 0x7f05a0089990 con 0x7f058004dcc0 2024-09-21T09:20:59.546 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.535+0000 7fab2b7fe640 1 -- 172.21.15.78:0/3793099369 <== osd.0 v1:172.21.15.78:6808/1449157914 2 ==== command_reply(tid 2: 0 ) v1 ==== 8+0+11 (unknown 0 0 3659784045) 0x7fab44133a80 con 0x7fab3004dcc0 2024-09-21T09:20:59.546 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.535+0000 7f05a5b33640 1 -- 172.21.15.78:0/1553556118 >> v1:172.21.15.78:6800/3190796132 conn(0x7f058004dcc0 legacy=0x7f0580050140 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:59.546 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.535+0000 7f93f3fff640 1 -- 172.21.15.78:0/3317798998 <== osd.3 v1:172.21.15.78:6802/4276488585 2 ==== command_reply(tid 2: 0 ) v1 ==== 8+0+11 (unknown 0 0 3659784045) 0x7f940c06d8e0 con 0x7f93f804dc40 2024-09-21T09:20:59.546 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.535+0000 7fab4943a640 1 -- 172.21.15.78:0/3793099369 >> v1:172.21.15.78:6808/1449157914 conn(0x7fab3004dcc0 legacy=0x7fab30050140 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:59.547 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.535+0000 7f05a5b33640 1 -- 172.21.15.78:0/1553556118 >> v1:172.21.15.78:6816/2602406893 conn(0x7f0580047630 legacy=0x7f0580049af0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:59.547 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.535+0000 7f05a5b33640 1 -- 172.21.15.78:0/1553556118 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f05a0070b80 msgr2=0x7f05a0114ff0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:59.547 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.535+0000 7f05a5b33640 1 --2- 172.21.15.78:0/1553556118 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f05a0070b80 0x7f05a0114ff0 secure :-1 s=READY pgs=34 cs=0 l=1 rev1=1 crypto rx=0x7f0590000c00 tx=0x7f05900042f0 comp rx=0 tx=0).stop 2024-09-21T09:20:59.547 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.535+0000 7f059ffff640 1 -- 172.21.15.78:0/1553556118 reap_dead start 2024-09-21T09:20:59.548 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.535+0000 7fab4943a640 1 -- 172.21.15.78:0/3793099369 >> v1:172.21.15.78:6816/2602406893 conn(0x7fab30047630 legacy=0x7fab30049af0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:59.548 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.535+0000 7fab4943a640 1 -- 172.21.15.78:0/3793099369 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fab44071ab0 msgr2=0x7fab44132510 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:59.548 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.535+0000 7fab4943a640 1 --2- 172.21.15.78:0/3793099369 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fab44071ab0 0x7fab44132510 secure :-1 s=READY pgs=63 cs=0 l=1 rev1=1 crypto rx=0x7fab3400d950 tx=0x7fab3400de10 comp rx=0 tx=0).stop 2024-09-21T09:20:59.548 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.535+0000 7f05a5b33640 1 -- 172.21.15.78:0/1553556118 shutdown_connections 2024-09-21T09:20:59.548 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.535+0000 7f05a5b33640 1 -- 172.21.15.78:0/1553556118 >> 172.21.15.78:0/1553556118 conn(0x7f05a009c8a0 msgr2=0x7f05a0127e40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:20:59.548 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.535+0000 7fab437fe640 1 -- 172.21.15.78:0/3793099369 reap_dead start 2024-09-21T09:20:59.549 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.536+0000 7fab4943a640 1 -- 172.21.15.78:0/3793099369 shutdown_connections 2024-09-21T09:20:59.549 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.536+0000 7f05a5b33640 1 -- 172.21.15.78:0/1553556118 shutdown_connections 2024-09-21T09:20:59.549 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.536+0000 7f05a5b33640 1 -- 172.21.15.78:0/1553556118 wait complete. 2024-09-21T09:20:59.549 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.536+0000 7f94119b5640 1 -- 172.21.15.78:0/3317798998 >> v1:172.21.15.78:6802/4276488585 conn(0x7f93f804dc40 legacy=0x7f93f80500c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:59.550 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.536+0000 7fab4943a640 1 -- 172.21.15.78:0/3793099369 >> 172.21.15.78:0/3793099369 conn(0x7fab4409ed50 msgr2=0x7fab44099b50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:20:59.550 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.536+0000 7fab4943a640 1 -- 172.21.15.78:0/3793099369 shutdown_connections 2024-09-21T09:20:59.550 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.536+0000 7fab4943a640 1 -- 172.21.15.78:0/3793099369 wait complete. 2024-09-21T09:20:59.550 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.536+0000 7f94119b5640 1 -- 172.21.15.78:0/3317798998 >> v1:172.21.15.78:6816/2602406893 conn(0x7f93f80475b0 legacy=0x7f93f8049a70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:59.550 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.536+0000 7f94119b5640 1 -- 172.21.15.78:0/3317798998 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f940c06f820 msgr2=0x7f940c1ba850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:59.551 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.536+0000 7f94119b5640 1 --2- 172.21.15.78:0/3317798998 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f940c06f820 0x7f940c1ba850 secure :-1 s=READY pgs=48 cs=0 l=1 rev1=1 crypto rx=0x7f940000fc80 tx=0x7f9400003b90 comp rx=0 tx=0).stop 2024-09-21T09:20:59.551 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.536+0000 7f940b7fe640 1 -- 172.21.15.78:0/3317798998 reap_dead start 2024-09-21T09:20:59.551 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.536+0000 7f94119b5640 1 -- 172.21.15.78:0/3317798998 shutdown_connections 2024-09-21T09:20:59.551 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.536+0000 7f94119b5640 1 -- 172.21.15.78:0/3317798998 >> 172.21.15.78:0/3317798998 conn(0x7f940c097c00 msgr2=0x7f940c098ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:20:59.551 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.536+0000 7f94119b5640 1 -- 172.21.15.78:0/3317798998 shutdown_connections 2024-09-21T09:20:59.551 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.536+0000 7f94119b5640 1 -- 172.21.15.78:0/3317798998 wait complete. 2024-09-21T09:20:59.551 INFO:teuthology.orchestra.run.smithi078.stdout:38654705670 2024-09-21T09:20:59.552 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.2 2024-09-21T09:20:59.553 INFO:teuthology.orchestra.run.smithi078.stdout:38654705668 2024-09-21T09:20:59.553 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.1 2024-09-21T09:20:59.554 INFO:teuthology.orchestra.run.smithi078.stdout:38654705668 2024-09-21T09:20:59.554 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.3 2024-09-21T09:20:59.555 INFO:teuthology.orchestra.run.smithi078.stdout:38654705668 2024-09-21T09:20:59.555 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.0 2024-09-21T09:20:59.698 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.697+0000 7f959fe20640 1 -- 172.21.15.78:0/4101125128 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f959812c330 msgr2=0x7f959812c7c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:59.698 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.697+0000 7f959fe20640 1 --2- 172.21.15.78:0/4101125128 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f959812c330 0x7f959812c7c0 secure :-1 s=READY pgs=49 cs=0 l=1 rev1=1 crypto rx=0x7f9590005fd0 tx=0x7f959002a770 comp rx=0 tx=0).stop 2024-09-21T09:20:59.699 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.697+0000 7fa3b205a640 1 -- 172.21.15.78:0/4201320138 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fa3ac06ff60 msgr2=0x7fa3ac128bd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:59.699 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.697+0000 7fa3b205a640 1 --2- 172.21.15.78:0/4201320138 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fa3ac06ff60 0x7fa3ac128bd0 secure :-1 s=READY pgs=35 cs=0 l=1 rev1=1 crypto rx=0x7fa3a0009860 tx=0x7fa3a002a390 comp rx=0 tx=0).stop 2024-09-21T09:20:59.699 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.697+0000 7fa3b205a640 1 -- 172.21.15.78:0/4201320138 shutdown_connections 2024-09-21T09:20:59.699 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.697+0000 7f959fe20640 1 -- 172.21.15.78:0/4101125128 shutdown_connections 2024-09-21T09:20:59.699 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.697+0000 7f959fe20640 1 --2- 172.21.15.78:0/4101125128 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f959812c330 0x7f959812c7c0 unknown :-1 s=CLOSED pgs=49 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:59.699 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.697+0000 7f959fe20640 1 --2- 172.21.15.78:0/4101125128 >> v2:172.21.15.78:3300/0 conn(0x7f9598070b80 0x7f959812bdf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:59.700 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.697+0000 7f959fe20640 1 --2- 172.21.15.78:0/4101125128 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f9598128d70 0x7f9598129150 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:59.700 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.697+0000 7f959fe20640 1 -- 172.21.15.78:0/4101125128 >> 172.21.15.78:0/4101125128 conn(0x7f959809c8a0 msgr2=0x7f959809ccb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:20:59.700 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.697+0000 7fa3b205a640 1 --2- 172.21.15.78:0/4201320138 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fa3ac06ff60 0x7fa3ac128bd0 unknown :-1 s=CLOSED pgs=35 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:59.700 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.697+0000 7f61271ed640 1 -- 172.21.15.78:0/1534725842 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f612012a440 msgr2=0x7f612012a8d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:59.700 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.697+0000 7f61271ed640 1 --2- 172.21.15.78:0/1534725842 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f612012a440 0x7f612012a8d0 secure :-1 s=READY pgs=66 cs=0 l=1 rev1=1 crypto rx=0x7f6118005fd0 tx=0x7f611802a820 comp rx=0 tx=0).stop 2024-09-21T09:20:59.700 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.697+0000 7fa3b205a640 1 --2- 172.21.15.78:0/4201320138 >> v2:172.21.15.78:3300/0 conn(0x7fa3ac129110 0x7fa3ac1295a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:59.701 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.697+0000 7f959fe20640 1 -- 172.21.15.78:0/4101125128 shutdown_connections 2024-09-21T09:20:59.701 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.697+0000 7fa3b205a640 1 --2- 172.21.15.78:0/4201320138 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fa3ac06f640 0x7fa3ac06fa20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:59.701 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.697+0000 7fa3b205a640 1 -- 172.21.15.78:0/4201320138 >> 172.21.15.78:0/4201320138 conn(0x7fa3ac09b9c0 msgr2=0x7fa3ac09bdd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:20:59.701 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.697+0000 7f61271ed640 1 -- 172.21.15.78:0/1534725842 shutdown_connections 2024-09-21T09:20:59.701 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.697+0000 7f61271ed640 1 --2- 172.21.15.78:0/1534725842 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f612012a440 0x7f612012a8d0 unknown :-1 s=CLOSED pgs=66 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:59.702 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.697+0000 7f61271ed640 1 --2- 172.21.15.78:0/1534725842 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f612006f150 0x7f612006f530 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:59.702 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.697+0000 7f61271ed640 1 -- 172.21.15.78:0/1534725842 >> 172.21.15.78:0/1534725842 conn(0x7f6120097190 msgr2=0x7f61200975a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:20:59.702 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.697+0000 7ff918795640 1 -- 172.21.15.78:0/1847547323 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7ff91006fe20 msgr2=0x7ff910070200 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:59.702 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.697+0000 7f959fe20640 1 -- 172.21.15.78:0/4101125128 wait complete. 2024-09-21T09:20:59.702 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.697+0000 7fa3b205a640 1 -- 172.21.15.78:0/4201320138 shutdown_connections 2024-09-21T09:20:59.703 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.697+0000 7ff918795640 1 --2- 172.21.15.78:0/1847547323 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7ff91006fe20 0x7ff910070200 secure :-1 s=READY pgs=67 cs=0 l=1 rev1=1 crypto rx=0x7ff90c005fd0 tx=0x7ff90c02a4b0 comp rx=0 tx=0).stop 2024-09-21T09:20:59.703 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.697+0000 7fa3b205a640 1 -- 172.21.15.78:0/4201320138 wait complete. 2024-09-21T09:20:59.703 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.697+0000 7f61271ed640 1 -- 172.21.15.78:0/1534725842 shutdown_connections 2024-09-21T09:20:59.704 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.697+0000 7ff918795640 1 -- 172.21.15.78:0/1847547323 shutdown_connections 2024-09-21T09:20:59.704 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.697+0000 7ff918795640 1 --2- 172.21.15.78:0/1847547323 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7ff910096080 0x7ff910131100 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:59.704 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.697+0000 7ff918795640 1 --2- 172.21.15.78:0/1847547323 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7ff91006fe20 0x7ff910070200 unknown :-1 s=CLOSED pgs=67 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:59.704 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.697+0000 7ff918795640 1 -- 172.21.15.78:0/1847547323 >> 172.21.15.78:0/1847547323 conn(0x7ff910098500 msgr2=0x7ff910098910 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:20:59.704 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.697+0000 7ff918795640 1 -- 172.21.15.78:0/1847547323 shutdown_connections 2024-09-21T09:20:59.704 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.697+0000 7ff918795640 1 -- 172.21.15.78:0/1847547323 wait complete. 2024-09-21T09:20:59.704 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.698+0000 7f61271ed640 1 -- 172.21.15.78:0/1534725842 wait complete. 2024-09-21T09:20:59.705 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.698+0000 7ff918795640 1 Processor -- start 2024-09-21T09:20:59.705 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.698+0000 7ff918795640 1 -- start start 2024-09-21T09:20:59.705 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.699+0000 7f959fe20640 1 Processor -- start 2024-09-21T09:20:59.705 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.699+0000 7fa3b205a640 1 Processor -- start 2024-09-21T09:20:59.705 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.699+0000 7f61271ed640 1 Processor -- start 2024-09-21T09:20:59.705 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.699+0000 7f61271ed640 1 -- start start 2024-09-21T09:20:59.706 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.699+0000 7f959fe20640 1 -- start start 2024-09-21T09:20:59.706 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.699+0000 7fa3b205a640 1 -- start start 2024-09-21T09:20:59.708 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.699+0000 7ff918795640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7ff91006fe20 0x7ff91012a0c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:59.708 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.699+0000 7ff918795640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7ff910070740 0x7ff91012a600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:59.708 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.699+0000 7ff918795640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7ff910096080 0x7ff91012ab40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:59.708 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.699+0000 7ff918795640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff91012b170 con 0x7ff910096080 2024-09-21T09:20:59.708 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.699+0000 7ff915d30640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7ff910070740 0x7ff91012a600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:59.708 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.699+0000 7ff918795640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff910096c60 con 0x7ff910070740 2024-09-21T09:20:59.708 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.699+0000 7ff918795640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 v1 -- 0x7ff91012c440 con 0x7ff91006fe20 2024-09-21T09:20:59.708 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.699+0000 7ff915d30640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7ff910070740 0x7ff91012a600 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3301/0 says I am v2:172.21.15.78:32986/0 (socket says 172.21.15.78:32986) 2024-09-21T09:20:59.708 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.699+0000 7ff915d30640 1 -- 172.21.15.78:0/1476419947 learned_addr learned my addr 172.21.15.78:0/1476419947 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:20:59.708 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.699+0000 7ff916531640 1 --2- 172.21.15.78:0/1476419947 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7ff91006fe20 0x7ff91012a0c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:59.708 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.699+0000 7ff916d32640 1 --2- 172.21.15.78:0/1476419947 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7ff910096080 0x7ff91012ab40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:59.708 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.699+0000 7ff916531640 1 -- 172.21.15.78:0/1476419947 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7ff910070740 msgr2=0x7ff91012a600 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:59.708 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.699+0000 7ff916531640 1 --2- 172.21.15.78:0/1476419947 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7ff910070740 0x7ff91012a600 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:59.709 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.699+0000 7ff916531640 1 -- 172.21.15.78:0/1476419947 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7ff910096080 msgr2=0x7ff91012ab40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:59.709 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.699+0000 7ff916531640 1 --2- 172.21.15.78:0/1476419947 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7ff910096080 0x7ff91012ab40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:59.709 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.699+0000 7ff916531640 1 -- 172.21.15.78:0/1476419947 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff90c009560 con 0x7ff91006fe20 2024-09-21T09:20:59.709 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.700+0000 7fa3b205a640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fa3ac06f640 0x7fa3ac1bdda0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:59.709 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.700+0000 7ff916531640 1 --2- 172.21.15.78:0/1476419947 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7ff91006fe20 0x7ff91012a0c0 secure :-1 s=READY pgs=36 cs=0 l=1 rev1=1 crypto rx=0x7ff90c031000 tx=0x7ff90c003910 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:20:59.709 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.699+0000 7f61271ed640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f612006f150 0x7f61201c20a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:59.710 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.700+0000 7fa3ab7fe640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fa3ac06f640 0x7fa3ac1bdda0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:59.710 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.699+0000 7f6124f89640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f612006f150 0x7f61201c20a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:59.710 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.699+0000 7f6124f89640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f612006f150 0x7f61201c20a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3300/0 says I am v2:172.21.15.78:37650/0 (socket says 172.21.15.78:37650) 2024-09-21T09:20:59.710 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.699+0000 7f61271ed640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f612006fa70 0x7f61201c25e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:59.710 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.700+0000 7f6124f89640 1 -- 172.21.15.78:0/4213085535 learned_addr learned my addr 172.21.15.78:0/4213085535 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:20:59.710 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.700+0000 7f61271ed640 1 --2- 172.21.15.78:0/4213085535 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f612012a440 0x7f612006ccd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:59.711 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.700+0000 7f611ffff640 1 --2- 172.21.15.78:0/4213085535 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f612006fa70 0x7f61201c25e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:59.711 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.700+0000 7f612578a640 1 --2- 172.21.15.78:0/4213085535 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f612012a440 0x7f612006ccd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:59.711 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.700+0000 7f61271ed640 1 -- 172.21.15.78:0/4213085535 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6120089140 con 0x7f612006f150 2024-09-21T09:20:59.711 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.700+0000 7f61271ed640 1 -- 172.21.15.78:0/4213085535 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6120096700 con 0x7f612006fa70 2024-09-21T09:20:59.711 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.700+0000 7f61271ed640 1 -- 172.21.15.78:0/4213085535 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 v1 -- 0x7f6120096ff0 con 0x7f612012a440 2024-09-21T09:20:59.711 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.700+0000 7ff906ffd640 1 -- 172.21.15.78:0/1476419947 <== mon.2 v2:172.21.15.78:3302/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff90c03c470 con 0x7ff91006fe20 2024-09-21T09:20:59.711 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.700+0000 7ff906ffd640 1 -- 172.21.15.78:0/1476419947 <== mon.2 v2:172.21.15.78:3302/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7ff90c03c610 con 0x7ff91006fe20 2024-09-21T09:20:59.711 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.700+0000 7ff906ffd640 1 -- 172.21.15.78:0/1476419947 <== mon.2 v2:172.21.15.78:3302/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff90c03ca20 con 0x7ff91006fe20 2024-09-21T09:20:59.711 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.700+0000 7ff918795640 1 -- 172.21.15.78:0/1476419947 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff910125d40 con 0x7ff91006fe20 2024-09-21T09:20:59.712 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.700+0000 7fa3ab7fe640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fa3ac06f640 0x7fa3ac1bdda0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3300/0 says I am v2:172.21.15.78:37658/0 (socket says 172.21.15.78:37658) 2024-09-21T09:20:59.712 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.700+0000 7f959fe20640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f9598070b80 0x7f959812fd40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:59.712 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.700+0000 7fa3ab7fe640 1 -- 172.21.15.78:0/3211669550 learned_addr learned my addr 172.21.15.78:0/3211669550 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:20:59.712 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.700+0000 7fa3b205a640 1 --2- 172.21.15.78:0/3211669550 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fa3ac06ff60 0x7fa3ac1be2e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:59.713 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.700+0000 7ff918795640 1 -- 172.21.15.78:0/1476419947 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff910125270 con 0x7ff91006fe20 2024-09-21T09:20:59.713 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.700+0000 7f959fe20640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f9598128d70 0x7f9598130280 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:59.713 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.700+0000 7fa3b205a640 1 --2- 172.21.15.78:0/3211669550 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fa3ac129110 0x7fa3ac1148c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:59.714 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.700+0000 7f6124f89640 1 -- 172.21.15.78:0/4213085535 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f612012a440 msgr2=0x7f612006ccd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:59.714 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.700+0000 7f6124f89640 1 --2- 172.21.15.78:0/4213085535 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f612012a440 0x7f612006ccd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:59.714 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.700+0000 7f6124f89640 1 -- 172.21.15.78:0/4213085535 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f612006fa70 msgr2=0x7f61201c25e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:59.714 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.700+0000 7f6124f89640 1 --2- 172.21.15.78:0/4213085535 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f612006fa70 0x7f61201c25e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:59.714 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.700+0000 7f6124f89640 1 -- 172.21.15.78:0/4213085535 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6118009840 con 0x7f612006f150 2024-09-21T09:20:59.714 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.700+0000 7fa3aaffd640 1 --2- 172.21.15.78:0/3211669550 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fa3ac06ff60 0x7fa3ac1be2e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:59.715 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.700+0000 7f959dbbc640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f9598070b80 0x7f959812fd40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:59.715 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.700+0000 7f959dbbc640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f9598070b80 0x7f959812fd40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3301/0 says I am v2:172.21.15.78:33010/0 (socket says 172.21.15.78:33010) 2024-09-21T09:20:59.715 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.701+0000 7f959dbbc640 1 -- 172.21.15.78:0/3516124418 learned_addr learned my addr 172.21.15.78:0/3516124418 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:20:59.715 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.701+0000 7fa3b205a640 1 -- 172.21.15.78:0/3211669550 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa3ac117d60 con 0x7fa3ac06f640 2024-09-21T09:20:59.715 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.701+0000 7fa3abfff640 1 --2- 172.21.15.78:0/3211669550 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fa3ac129110 0x7fa3ac1148c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:59.715 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.701+0000 7fa3b205a640 1 -- 172.21.15.78:0/3211669550 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa3ac118070 con 0x7fa3ac06ff60 2024-09-21T09:20:59.716 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.701+0000 7f959fe20640 1 --2- 172.21.15.78:0/3516124418 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f959812c330 0x7f95981307c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:20:59.716 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.701+0000 7fa3b205a640 1 -- 172.21.15.78:0/3211669550 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 v1 -- 0x7fa3ac116c80 con 0x7fa3ac129110 2024-09-21T09:20:59.716 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.701+0000 7f959d3bb640 1 --2- 172.21.15.78:0/3516124418 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f9598128d70 0x7f9598130280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:59.716 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.701+0000 7f959fe20640 1 -- 172.21.15.78:0/3516124418 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9598132f80 con 0x7f9598128d70 2024-09-21T09:20:59.716 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.701+0000 7f959fe20640 1 -- 172.21.15.78:0/3516124418 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9598126580 con 0x7f9598070b80 2024-09-21T09:20:59.716 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.701+0000 7f959fe20640 1 -- 172.21.15.78:0/3516124418 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 v1 -- 0x7f9598131c70 con 0x7f959812c330 2024-09-21T09:20:59.717 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.701+0000 7f959e3bd640 1 --2- 172.21.15.78:0/3516124418 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f959812c330 0x7f95981307c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:20:59.717 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.701+0000 7f6124f89640 1 --2- 172.21.15.78:0/4213085535 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f612006f150 0x7f61201c20a0 secure :-1 s=READY pgs=68 cs=0 l=1 rev1=1 crypto rx=0x7f611400ea20 tx=0x7f611400eee0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:20:59.717 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.701+0000 7fa3aaffd640 1 -- 172.21.15.78:0/3211669550 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fa3ac129110 msgr2=0x7fa3ac1148c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:59.718 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.701+0000 7fa3aaffd640 1 --2- 172.21.15.78:0/3211669550 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fa3ac129110 0x7fa3ac1148c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:59.718 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.701+0000 7fa3aaffd640 1 -- 172.21.15.78:0/3211669550 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fa3ac06f640 msgr2=0x7fa3ac1bdda0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:59.718 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.701+0000 7fa3aaffd640 1 --2- 172.21.15.78:0/3211669550 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fa3ac06f640 0x7fa3ac1bdda0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:59.718 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.701+0000 7fa3aaffd640 1 -- 172.21.15.78:0/3211669550 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa3a0009490 con 0x7fa3ac06ff60 2024-09-21T09:20:59.718 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.701+0000 7f612578a640 1 --2- 172.21.15.78:0/4213085535 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f612012a440 0x7f612006ccd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:20:59.718 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.701+0000 7f611d7fa640 1 -- 172.21.15.78:0/4213085535 <== mon.0 v2:172.21.15.78:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f611400cd20 con 0x7f612006f150 2024-09-21T09:20:59.718 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.701+0000 7f611d7fa640 1 -- 172.21.15.78:0/4213085535 <== mon.0 v2:172.21.15.78:3300/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7f6114010430 con 0x7f612006f150 2024-09-21T09:20:59.718 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.701+0000 7f611d7fa640 1 -- 172.21.15.78:0/4213085535 <== mon.0 v2:172.21.15.78:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6114010800 con 0x7f612006f150 2024-09-21T09:20:59.719 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.701+0000 7fa3aaffd640 1 --2- 172.21.15.78:0/3211669550 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fa3ac06ff60 0x7fa3ac1be2e0 secure :-1 s=READY pgs=50 cs=0 l=1 rev1=1 crypto rx=0x7fa3a0009830 tx=0x7fa3a0033b30 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:20:59.719 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.701+0000 7f959dbbc640 1 -- 172.21.15.78:0/3516124418 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f959812c330 msgr2=0x7f95981307c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:59.719 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.701+0000 7f959dbbc640 1 --2- 172.21.15.78:0/3516124418 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f959812c330 0x7f95981307c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:59.719 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.701+0000 7f959dbbc640 1 -- 172.21.15.78:0/3516124418 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f9598128d70 msgr2=0x7f9598130280 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:59.719 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.701+0000 7f959dbbc640 1 --2- 172.21.15.78:0/3516124418 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f9598128d70 0x7f9598130280 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:59.719 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.701+0000 7fa3abfff640 1 --2- 172.21.15.78:0/3211669550 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fa3ac129110 0x7fa3ac1148c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-09-21T09:20:59.720 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.702+0000 7f959dbbc640 1 -- 172.21.15.78:0/3516124418 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9588009490 con 0x7f9598070b80 2024-09-21T09:20:59.720 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.702+0000 7f959e3bd640 1 --2- 172.21.15.78:0/3516124418 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f959812c330 0x7f95981307c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-09-21T09:20:59.720 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.702+0000 7f61271ed640 1 -- 172.21.15.78:0/4213085535 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f612006e470 con 0x7f612006f150 2024-09-21T09:20:59.720 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.702+0000 7f959d3bb640 1 --2- 172.21.15.78:0/3516124418 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f9598128d70 0x7f9598130280 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:20:59.720 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.702+0000 7f959dbbc640 1 --2- 172.21.15.78:0/3516124418 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f9598070b80 0x7f959812fd40 secure :-1 s=READY pgs=51 cs=0 l=1 rev1=1 crypto rx=0x7f959400e9c0 tx=0x7f959400ee80 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:20:59.721 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.702+0000 7fa393fff640 1 -- 172.21.15.78:0/3211669550 <== mon.1 v2:172.21.15.78:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa3a00037f0 con 0x7fa3ac06ff60 2024-09-21T09:20:59.721 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.702+0000 7fa393fff640 1 -- 172.21.15.78:0/3211669550 <== mon.1 v2:172.21.15.78:3301/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7fa3a0003a60 con 0x7fa3ac06ff60 2024-09-21T09:20:59.721 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.702+0000 7fa393fff640 1 -- 172.21.15.78:0/3211669550 <== mon.1 v2:172.21.15.78:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa3a0003e70 con 0x7fa3ac06ff60 2024-09-21T09:20:59.721 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.702+0000 7ff906ffd640 1 -- 172.21.15.78:0/1476419947 <== mon.2 v2:172.21.15.78:3302/0 4 ==== mgrmap(e 5) v1 ==== 55212+0+0 (secure 0 0 0) 0x7ff90c03cb80 con 0x7ff91006fe20 2024-09-21T09:20:59.721 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.702+0000 7f958e7fc640 1 -- 172.21.15.78:0/3516124418 <== mon.1 v2:172.21.15.78:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f959400ccc0 con 0x7f9598070b80 2024-09-21T09:20:59.721 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.702+0000 7f958e7fc640 1 -- 172.21.15.78:0/3516124418 <== mon.1 v2:172.21.15.78:3301/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7f959400ce20 con 0x7f9598070b80 2024-09-21T09:20:59.721 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.702+0000 7f958e7fc640 1 -- 172.21.15.78:0/3516124418 <== mon.1 v2:172.21.15.78:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9594005280 con 0x7f9598070b80 2024-09-21T09:20:59.722 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.703+0000 7f959fe20640 1 -- 172.21.15.78:0/3516124418 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9590009840 con 0x7f9598070b80 2024-09-21T09:20:59.722 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.703+0000 7f959fe20640 1 -- 172.21.15.78:0/3516124418 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9598132670 con 0x7f9598070b80 2024-09-21T09:20:59.722 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.702+0000 7fa3b205a640 1 -- 172.21.15.78:0/3211669550 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa3ac1be820 con 0x7fa3ac06ff60 2024-09-21T09:20:59.722 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.702+0000 7fa3b205a640 1 -- 172.21.15.78:0/3211669550 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa3ac114e00 con 0x7fa3ac06ff60 2024-09-21T09:20:59.722 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.702+0000 7f61271ed640 1 -- 172.21.15.78:0/4213085535 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6120089620 con 0x7f612006f150 2024-09-21T09:20:59.722 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.702+0000 7ff906ffd640 1 -- 172.21.15.78:0/1476419947 <== mon.2 v2:172.21.15.78:3302/0 5 ==== osd_map(17..17 src has 1..17) v4 ==== 2868+0+0 (secure 0 0 0) 0x7ff90c07de10 con 0x7ff91006fe20 2024-09-21T09:20:59.723 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.703+0000 7ff918795640 1 -- 172.21.15.78:0/1476419947 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff91004e370 con 0x7ff91006fe20 2024-09-21T09:20:59.723 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.703+0000 7f611d7fa640 1 -- 172.21.15.78:0/4213085535 <== mon.0 v2:172.21.15.78:3300/0 4 ==== mgrmap(e 5) v1 ==== 55212+0+0 (secure 0 0 0) 0x7f6114010960 con 0x7f612006f150 2024-09-21T09:20:59.723 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.703+0000 7fa3b205a640 1 -- 172.21.15.78:0/3211669550 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa3ac1be9b0 con 0x7fa3ac06ff60 2024-09-21T09:20:59.723 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.704+0000 7f959fe20640 1 -- 172.21.15.78:0/3516124418 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9598129150 con 0x7f9598070b80 2024-09-21T09:20:59.723 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.704+0000 7fa393fff640 1 -- 172.21.15.78:0/3211669550 <== mon.1 v2:172.21.15.78:3301/0 4 ==== mgrmap(e 5) v1 ==== 55212+0+0 (secure 0 0 0) 0x7fa3a003e660 con 0x7fa3ac06ff60 2024-09-21T09:20:59.724 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.703+0000 7f61271ed640 1 -- 172.21.15.78:0/4213085535 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6120071c40 con 0x7f612006f150 2024-09-21T09:20:59.724 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.704+0000 7f611d7fa640 1 -- 172.21.15.78:0/4213085535 <== mon.0 v2:172.21.15.78:3300/0 5 ==== osd_map(17..17 src has 1..17) v4 ==== 2868+0+0 (secure 0 0 0) 0x7f6114014070 con 0x7f612006f150 2024-09-21T09:20:59.724 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.705+0000 7f958e7fc640 1 -- 172.21.15.78:0/3516124418 <== mon.1 v2:172.21.15.78:3301/0 4 ==== mgrmap(e 5) v1 ==== 55212+0+0 (secure 0 0 0) 0x7f95940053e0 con 0x7f9598070b80 2024-09-21T09:20:59.724 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.705+0000 7fa393fff640 1 -- 172.21.15.78:0/3211669550 <== mon.1 v2:172.21.15.78:3301/0 5 ==== osd_map(17..17 src has 1..17) v4 ==== 2868+0+0 (secure 0 0 0) 0x7fa3a007e490 con 0x7fa3ac06ff60 2024-09-21T09:20:59.724 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.705+0000 7f958e7fc640 1 -- 172.21.15.78:0/3516124418 <== mon.1 v2:172.21.15.78:3301/0 5 ==== osd_map(17..17 src has 1..17) v4 ==== 2868+0+0 (secure 0 0 0) 0x7f9594014070 con 0x7f9598070b80 2024-09-21T09:20:59.725 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.708+0000 7fa393fff640 1 -- 172.21.15.78:0/3211669550 <== mon.1 v2:172.21.15.78:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+146310 (secure 0 0 0) 0x7fa3a00326a0 con 0x7fa3ac06ff60 2024-09-21T09:20:59.725 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.711+0000 7ff906ffd640 1 -- 172.21.15.78:0/1476419947 <== mon.2 v2:172.21.15.78:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+146310 (secure 0 0 0) 0x7ff91004e370 con 0x7ff91006fe20 2024-09-21T09:20:59.725 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.711+0000 7f958e7fc640 1 -- 172.21.15.78:0/3516124418 <== mon.1 v2:172.21.15.78:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+146310 (secure 0 0 0) 0x7f959403b240 con 0x7f9598070b80 2024-09-21T09:20:59.725 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.713+0000 7f611d7fa640 1 -- 172.21.15.78:0/4213085535 <== mon.0 v2:172.21.15.78:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+146310 (secure 0 0 0) 0x7f611403af90 con 0x7f612006f150 2024-09-21T09:20:59.851 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.849+0000 7fa3b205a640 1 -- 172.21.15.78:0/3211669550 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 2} v 0) v1 -- 0x7fa3ac06c9a0 con 0x7fa3ac06ff60 2024-09-21T09:20:59.851 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.849+0000 7fa393fff640 1 -- 172.21.15.78:0/3211669550 <== mon.1 v2:172.21.15.78:3301/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 2}]=0 v0) v1 ==== 74+0+12 (secure 0 0 0) 0x7fa3a0042020 con 0x7fa3ac06ff60 2024-09-21T09:20:59.851 INFO:teuthology.orchestra.run.smithi078.stdout:38654705669 2024-09-21T09:20:59.854 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.852+0000 7fa3b205a640 1 -- 172.21.15.78:0/3211669550 >> v1:172.21.15.78:6816/2602406893 conn(0x7fa39804ba40 legacy=0x7fa39804df00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:59.854 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.852+0000 7fa3b205a640 1 -- 172.21.15.78:0/3211669550 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fa3ac06ff60 msgr2=0x7fa3ac1be2e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:59.854 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.852+0000 7fa3b205a640 1 --2- 172.21.15.78:0/3211669550 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fa3ac06ff60 0x7fa3ac1be2e0 secure :-1 s=READY pgs=50 cs=0 l=1 rev1=1 crypto rx=0x7fa3a0009830 tx=0x7fa3a0033b30 comp rx=0 tx=0).stop 2024-09-21T09:20:59.854 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.852+0000 7fa3b205a640 1 -- 172.21.15.78:0/3211669550 shutdown_connections 2024-09-21T09:20:59.854 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.852+0000 7fa3b205a640 1 --2- 172.21.15.78:0/3211669550 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fa3ac129110 0x7fa3ac1148c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:59.854 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.852+0000 7fa3b205a640 1 --2- 172.21.15.78:0/3211669550 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fa3ac06ff60 0x7fa3ac1be2e0 unknown :-1 s=CLOSED pgs=50 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:59.854 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.852+0000 7fa3b205a640 1 --2- 172.21.15.78:0/3211669550 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fa3ac06f640 0x7fa3ac1bdda0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:59.854 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.852+0000 7fa3b205a640 1 -- 172.21.15.78:0/3211669550 >> 172.21.15.78:0/3211669550 conn(0x7fa3ac09b9c0 msgr2=0x7fa3ac06e1d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:20:59.855 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.853+0000 7fa3b205a640 1 -- 172.21.15.78:0/3211669550 shutdown_connections 2024-09-21T09:20:59.855 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.853+0000 7fa3b205a640 1 -- 172.21.15.78:0/3211669550 wait complete. 2024-09-21T09:20:59.855 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.853+0000 7f959fe20640 1 -- 172.21.15.78:0/3516124418 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 1} v 0) v1 -- 0x7f9598124970 con 0x7f9598070b80 2024-09-21T09:20:59.855 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.854+0000 7f958e7fc640 1 -- 172.21.15.78:0/3516124418 <== mon.1 v2:172.21.15.78:3301/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 1}]=0 v0) v1 ==== 74+0+12 (secure 0 0 0) 0x7f959403e860 con 0x7f9598070b80 2024-09-21T09:20:59.856 INFO:teuthology.orchestra.run.smithi078.stdout:38654705667 2024-09-21T09:20:59.856 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.855+0000 7f959fe20640 1 -- 172.21.15.78:0/3516124418 >> v1:172.21.15.78:6816/2602406893 conn(0x7f957c047560 legacy=0x7f957c049a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:59.857 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.855+0000 7f959fe20640 1 -- 172.21.15.78:0/3516124418 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f9598070b80 msgr2=0x7f959812fd40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:59.857 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.855+0000 7f61271ed640 1 -- 172.21.15.78:0/4213085535 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 3} v 0) v1 -- 0x7f612006f530 con 0x7f612006f150 2024-09-21T09:20:59.858 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.855+0000 7f959fe20640 1 --2- 172.21.15.78:0/3516124418 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f9598070b80 0x7f959812fd40 secure :-1 s=READY pgs=51 cs=0 l=1 rev1=1 crypto rx=0x7f959400e9c0 tx=0x7f959400ee80 comp rx=0 tx=0).stop 2024-09-21T09:20:59.858 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.855+0000 7f959fe20640 1 -- 172.21.15.78:0/3516124418 shutdown_connections 2024-09-21T09:20:59.858 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.855+0000 7f959fe20640 1 --2- 172.21.15.78:0/3516124418 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f959812c330 0x7f95981307c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:59.858 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.855+0000 7f959fe20640 1 --2- 172.21.15.78:0/3516124418 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f9598128d70 0x7f9598130280 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:59.858 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.855+0000 7f959fe20640 1 --2- 172.21.15.78:0/3516124418 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f9598070b80 0x7f959812fd40 unknown :-1 s=CLOSED pgs=51 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:59.858 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.855+0000 7f959fe20640 1 -- 172.21.15.78:0/3516124418 >> 172.21.15.78:0/3516124418 conn(0x7f959809c8a0 msgr2=0x7f9598127e40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:20:59.858 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.856+0000 7f959fe20640 1 -- 172.21.15.78:0/3516124418 shutdown_connections 2024-09-21T09:20:59.858 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.856+0000 7f959fe20640 1 -- 172.21.15.78:0/3516124418 wait complete. 2024-09-21T09:20:59.859 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.856+0000 7f611d7fa640 1 -- 172.21.15.78:0/4213085535 <== mon.0 v2:172.21.15.78:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 3}]=0 v0) v1 ==== 74+0+12 (secure 0 0 0) 0x7f611403e5b0 con 0x7f612006f150 2024-09-21T09:20:59.859 INFO:teuthology.orchestra.run.smithi078.stdout:38654705667 2024-09-21T09:20:59.859 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.857+0000 7ff918795640 1 -- 172.21.15.78:0/1476419947 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 0} v 0) v1 -- 0x7ff91004e370 con 0x7ff91006fe20 2024-09-21T09:20:59.859 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.858+0000 7ff906ffd640 1 -- 172.21.15.78:0/1476419947 <== mon.2 v2:172.21.15.78:3302/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 0}]=0 v0) v1 ==== 74+0+12 (secure 0 0 0) 0x7ff90c035020 con 0x7ff91006fe20 2024-09-21T09:20:59.860 INFO:teuthology.orchestra.run.smithi078.stdout:38654705667 2024-09-21T09:20:59.860 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.858+0000 7f61271ed640 1 -- 172.21.15.78:0/4213085535 >> v1:172.21.15.78:6816/2602406893 conn(0x7f6104047630 legacy=0x7f6104049af0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:59.860 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.858+0000 7f61271ed640 1 -- 172.21.15.78:0/4213085535 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f612006f150 msgr2=0x7f61201c20a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:59.861 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.858+0000 7f61271ed640 1 --2- 172.21.15.78:0/4213085535 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f612006f150 0x7f61201c20a0 secure :-1 s=READY pgs=68 cs=0 l=1 rev1=1 crypto rx=0x7f611400ea20 tx=0x7f611400eee0 comp rx=0 tx=0).stop 2024-09-21T09:20:59.861 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.858+0000 7f61271ed640 1 -- 172.21.15.78:0/4213085535 shutdown_connections 2024-09-21T09:20:59.861 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.858+0000 7f61271ed640 1 --2- 172.21.15.78:0/4213085535 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f612012a440 0x7f612006ccd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:59.861 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.858+0000 7f61271ed640 1 --2- 172.21.15.78:0/4213085535 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f612006fa70 0x7f61201c25e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:59.861 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.858+0000 7f61271ed640 1 --2- 172.21.15.78:0/4213085535 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f612006f150 0x7f61201c20a0 unknown :-1 s=CLOSED pgs=68 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:59.861 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.858+0000 7f61271ed640 1 -- 172.21.15.78:0/4213085535 >> 172.21.15.78:0/4213085535 conn(0x7f6120097190 msgr2=0x7f612009ba00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:20:59.861 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.858+0000 7f61271ed640 1 -- 172.21.15.78:0/4213085535 shutdown_connections 2024-09-21T09:20:59.861 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.858+0000 7f61271ed640 1 -- 172.21.15.78:0/4213085535 wait complete. 2024-09-21T09:20:59.862 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.861+0000 7ff918795640 1 -- 172.21.15.78:0/1476419947 >> v1:172.21.15.78:6816/2602406893 conn(0x7ff8f404b970 legacy=0x7ff8f404de30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:59.862 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.861+0000 7ff918795640 1 -- 172.21.15.78:0/1476419947 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7ff91006fe20 msgr2=0x7ff91012a0c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:20:59.863 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.861+0000 7ff918795640 1 --2- 172.21.15.78:0/1476419947 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7ff91006fe20 0x7ff91012a0c0 secure :-1 s=READY pgs=36 cs=0 l=1 rev1=1 crypto rx=0x7ff90c031000 tx=0x7ff90c003910 comp rx=0 tx=0).stop 2024-09-21T09:20:59.863 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.861+0000 7ff918795640 1 -- 172.21.15.78:0/1476419947 shutdown_connections 2024-09-21T09:20:59.863 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.861+0000 7ff918795640 1 --2- 172.21.15.78:0/1476419947 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7ff910096080 0x7ff91012ab40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:59.863 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.861+0000 7ff918795640 1 --2- 172.21.15.78:0/1476419947 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7ff910070740 0x7ff91012a600 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:59.863 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.861+0000 7ff918795640 1 --2- 172.21.15.78:0/1476419947 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7ff91006fe20 0x7ff91012a0c0 unknown :-1 s=CLOSED pgs=36 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:20:59.863 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.861+0000 7ff918795640 1 -- 172.21.15.78:0/1476419947 >> 172.21.15.78:0/1476419947 conn(0x7ff910098500 msgr2=0x7ff910093ea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:20:59.863 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.861+0000 7ff918795640 1 -- 172.21.15.78:0/1476419947 shutdown_connections 2024-09-21T09:20:59.863 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:20:59.861+0000 7ff918795640 1 -- 172.21.15.78:0/1476419947 wait complete. 2024-09-21T09:20:59.871 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705668 got 38654705667 for osd.1 2024-09-21T09:20:59.872 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705670 got 38654705669 for osd.2 2024-09-21T09:20:59.873 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705668 got 38654705667 for osd.3 2024-09-21T09:20:59.876 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705668 got 38654705667 for osd.0 2024-09-21T09:21:00.872 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.1 2024-09-21T09:21:00.872 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.2 2024-09-21T09:21:00.873 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.3 2024-09-21T09:21:00.877 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.0 2024-09-21T09:21:00.989 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.987+0000 7f519989c640 1 -- 172.21.15.78:0/2698089010 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f5194129e40 msgr2=0x7f519412a2f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:21:00.989 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.987+0000 7f519989c640 1 --2- 172.21.15.78:0/2698089010 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f5194129e40 0x7f519412a2f0 secure :-1 s=READY pgs=70 cs=0 l=1 rev1=1 crypto rx=0x7f518c005fd0 tx=0x7f518c02a770 comp rx=0 tx=0).stop 2024-09-21T09:21:00.989 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.987+0000 7f519989c640 1 -- 172.21.15.78:0/2698089010 shutdown_connections 2024-09-21T09:21:00.989 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.987+0000 7f519989c640 1 --2- 172.21.15.78:0/2698089010 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f5194097130 0x7f519406f650 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:00.989 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.987+0000 7f519989c640 1 --2- 172.21.15.78:0/2698089010 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f5194129e40 0x7f519412a2f0 unknown :-1 s=CLOSED pgs=70 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:00.989 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.987+0000 7f519989c640 1 --2- 172.21.15.78:0/2698089010 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f519406ed30 0x7f519406f110 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:00.989 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.987+0000 7f519989c640 1 -- 172.21.15.78:0/2698089010 >> 172.21.15.78:0/2698089010 conn(0x7f5194097c00 msgr2=0x7f5194098010 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:21:00.990 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.987+0000 7f6aa08e5640 1 -- 172.21.15.78:0/715914692 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f6a9c071c80 msgr2=0x7f6a9c0720b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:21:00.990 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.987+0000 7f6aa08e5640 1 --2- 172.21.15.78:0/715914692 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f6a9c071c80 0x7f6a9c0720b0 secure :-1 s=READY pgs=71 cs=0 l=1 rev1=1 crypto rx=0x7f6a94005fd0 tx=0x7f6a9402a780 comp rx=0 tx=0).stop 2024-09-21T09:21:00.990 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.987+0000 7f519989c640 1 -- 172.21.15.78:0/2698089010 shutdown_connections 2024-09-21T09:21:00.990 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.987+0000 7f519989c640 1 -- 172.21.15.78:0/2698089010 wait complete. 2024-09-21T09:21:00.991 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.988+0000 7f6aa08e5640 1 -- 172.21.15.78:0/715914692 shutdown_connections 2024-09-21T09:21:00.991 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.988+0000 7f6aa08e5640 1 --2- 172.21.15.78:0/715914692 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f6a9c071c80 0x7f6a9c0720b0 unknown :-1 s=CLOSED pgs=71 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:00.991 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.988+0000 7f6aa08e5640 1 --2- 172.21.15.78:0/715914692 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f6a9c096c30 0x7f6a9c097010 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:00.991 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.988+0000 7f6aa08e5640 1 -- 172.21.15.78:0/715914692 >> 172.21.15.78:0/715914692 conn(0x7f6a9c09ed50 msgr2=0x7f6a9c09b220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:21:00.991 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.988+0000 7f6aa08e5640 1 -- 172.21.15.78:0/715914692 shutdown_connections 2024-09-21T09:21:00.991 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.988+0000 7f6aa08e5640 1 -- 172.21.15.78:0/715914692 wait complete. 2024-09-21T09:21:00.992 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.988+0000 7f519989c640 1 Processor -- start 2024-09-21T09:21:00.992 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.989+0000 7f519989c640 1 -- start start 2024-09-21T09:21:00.992 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.989+0000 7f6aa08e5640 1 Processor -- start 2024-09-21T09:21:00.992 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.989+0000 7f6aa08e5640 1 -- start start 2024-09-21T09:21:00.993 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.989+0000 7f519989c640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f519406ed30 0x7f519406cb20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:21:00.993 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.989+0000 7f519989c640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f5194097130 0x7f519406d060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:21:00.993 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.989+0000 7f519989c640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f5194129e40 0x7f519406d5a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:21:00.993 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.989+0000 7f6aa08e5640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f6a9c071c80 0x7f6a9c133520 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:21:00.993 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.989+0000 7f6aa08e5640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f6a9c096c30 0x7f6a9c133a60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:21:00.993 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.989+0000 7f6aa08e5640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f6a9c097550 0x7f6a9c133fa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:21:00.993 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.989+0000 7f6aa08e5640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6a9c08f720 con 0x7f6a9c096c30 2024-09-21T09:21:00.993 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.989+0000 7f6aa08e5640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6a9c08f160 con 0x7f6a9c097550 2024-09-21T09:21:00.994 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.989+0000 7f6aa08e5640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 v1 -- 0x7f6a9c126810 con 0x7f6a9c071c80 2024-09-21T09:21:00.994 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.989+0000 7f519989c640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f519408f0d0 con 0x7f5194129e40 2024-09-21T09:21:00.994 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.989+0000 7f519989c640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f51940900c0 con 0x7f519406ed30 2024-09-21T09:21:00.994 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.989+0000 7f519989c640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 v1 -- 0x7f519408f510 con 0x7f5194097130 2024-09-21T09:21:00.994 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.989+0000 7f51927fc640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f5194097130 0x7f519406d060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:21:00.994 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.989+0000 7f51927fc640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f5194097130 0x7f519406d060 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3302/0 says I am v2:172.21.15.78:35940/0 (socket says 172.21.15.78:35940) 2024-09-21T09:21:00.994 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.989+0000 7f51927fc640 1 -- 172.21.15.78:0/2487486260 learned_addr learned my addr 172.21.15.78:0/2487486260 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:21:00.994 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.989+0000 7f51937fe640 1 --2- 172.21.15.78:0/2487486260 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f5194129e40 0x7f519406d5a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:21:00.995 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.989+0000 7f6a99d9b640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f6a9c096c30 0x7f6a9c133a60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:21:00.995 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.990+0000 7f6a9a59c640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f6a9c071c80 0x7f6a9c133520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:21:00.995 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.990+0000 7f6a99d9b640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f6a9c096c30 0x7f6a9c133a60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3300/0 says I am v2:172.21.15.78:37724/0 (socket says 172.21.15.78:37724) 2024-09-21T09:21:00.995 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.990+0000 7f6a99d9b640 1 -- 172.21.15.78:0/2903294024 learned_addr learned my addr 172.21.15.78:0/2903294024 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:21:00.995 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.990+0000 7f6a9ad9d640 1 --2- 172.21.15.78:0/2903294024 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f6a9c097550 0x7f6a9c133fa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:21:00.996 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.990+0000 7f51927fc640 1 -- 172.21.15.78:0/2487486260 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f519406ed30 msgr2=0x7f519406cb20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:21:00.996 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.990+0000 7f5192ffd640 1 --2- 172.21.15.78:0/2487486260 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f519406ed30 0x7f519406cb20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:21:00.996 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.990+0000 7f51927fc640 1 --2- 172.21.15.78:0/2487486260 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f519406ed30 0x7f519406cb20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:00.996 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.990+0000 7f51927fc640 1 -- 172.21.15.78:0/2487486260 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f5194129e40 msgr2=0x7f519406d5a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:21:00.996 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.990+0000 7f51927fc640 1 --2- 172.21.15.78:0/2487486260 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f5194129e40 0x7f519406d5a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:00.996 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.990+0000 7f51927fc640 1 -- 172.21.15.78:0/2487486260 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5184009560 con 0x7f5194097130 2024-09-21T09:21:00.996 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.990+0000 7f51937fe640 1 --2- 172.21.15.78:0/2487486260 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f5194129e40 0x7f519406d5a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:21:00.996 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.990+0000 7f5192ffd640 1 --2- 172.21.15.78:0/2487486260 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f519406ed30 0x7f519406cb20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-09-21T09:21:00.996 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.990+0000 7f51927fc640 1 --2- 172.21.15.78:0/2487486260 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f5194097130 0x7f519406d060 secure :-1 s=READY pgs=37 cs=0 l=1 rev1=1 crypto rx=0x7f518800e980 tx=0x7f518800ee40 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:21:00.996 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.990+0000 7f517bfff640 1 -- 172.21.15.78:0/2487486260 <== mon.2 v2:172.21.15.78:3302/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f518800cd80 con 0x7f5194097130 2024-09-21T09:21:00.997 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.990+0000 7f6a9a59c640 1 -- 172.21.15.78:0/2903294024 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f6a9c097550 msgr2=0x7f6a9c133fa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:21:00.997 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.990+0000 7f6a9a59c640 1 --2- 172.21.15.78:0/2903294024 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f6a9c097550 0x7f6a9c133fa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:00.997 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.990+0000 7f6a9a59c640 1 -- 172.21.15.78:0/2903294024 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f6a9c096c30 msgr2=0x7f6a9c133a60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:21:00.997 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.990+0000 7f6a9a59c640 1 --2- 172.21.15.78:0/2903294024 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f6a9c096c30 0x7f6a9c133a60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:00.997 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.990+0000 7f6a9a59c640 1 -- 172.21.15.78:0/2903294024 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6a94009840 con 0x7f6a9c071c80 2024-09-21T09:21:00.997 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.990+0000 7f6a99d9b640 1 --2- 172.21.15.78:0/2903294024 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f6a9c096c30 0x7f6a9c133a60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:21:00.997 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.990+0000 7f6a9a59c640 1 --2- 172.21.15.78:0/2903294024 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f6a9c071c80 0x7f6a9c133520 secure :-1 s=READY pgs=38 cs=0 l=1 rev1=1 crypto rx=0x7f6a8c00e790 tx=0x7f6a8c00ec50 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:21:00.998 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.990+0000 7f517bfff640 1 -- 172.21.15.78:0/2487486260 <== mon.2 v2:172.21.15.78:3302/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7f5188010430 con 0x7f5194097130 2024-09-21T09:21:00.998 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.990+0000 7f517bfff640 1 -- 172.21.15.78:0/2487486260 <== mon.2 v2:172.21.15.78:3302/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5188010820 con 0x7f5194097130 2024-09-21T09:21:00.998 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.990+0000 7f519989c640 1 -- 172.21.15.78:0/2487486260 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f5188009490 con 0x7f5194097130 2024-09-21T09:21:00.998 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.991+0000 7f519989c640 1 -- 172.21.15.78:0/2487486260 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f518c009840 con 0x7f5194097130 2024-09-21T09:21:00.999 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.991+0000 7f6a8affd640 1 -- 172.21.15.78:0/2903294024 <== mon.2 v2:172.21.15.78:3302/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6a8c00ccc0 con 0x7f6a9c071c80 2024-09-21T09:21:00.999 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.991+0000 7f6a8affd640 1 -- 172.21.15.78:0/2903294024 <== mon.2 v2:172.21.15.78:3302/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7f6a8c00eea0 con 0x7f6a9c071c80 2024-09-21T09:21:00.999 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.991+0000 7f6aa08e5640 1 -- 172.21.15.78:0/2903294024 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6a9c0951b0 con 0x7f6a9c071c80 2024-09-21T09:21:00.999 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.991+0000 7f6a8affd640 1 -- 172.21.15.78:0/2903294024 <== mon.2 v2:172.21.15.78:3302/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6a8c010690 con 0x7f6a9c071c80 2024-09-21T09:21:00.999 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.991+0000 7f6aa08e5640 1 -- 172.21.15.78:0/2903294024 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6a9c0719f0 con 0x7f6a9c071c80 2024-09-21T09:21:00.999 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.992+0000 7f519989c640 1 -- 172.21.15.78:0/2487486260 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f519412a2f0 con 0x7f5194097130 2024-09-21T09:21:00.999 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.992+0000 7f517bfff640 1 -- 172.21.15.78:0/2487486260 <== mon.2 v2:172.21.15.78:3302/0 4 ==== mgrmap(e 5) v1 ==== 55212+0+0 (secure 0 0 0) 0x7f5188010980 con 0x7f5194097130 2024-09-21T09:21:01.000 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.992+0000 7f6aa08e5640 1 -- 172.21.15.78:0/2903294024 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6a9c0720b0 con 0x7f6a9c071c80 2024-09-21T09:21:01.000 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.992+0000 7f6a8affd640 1 -- 172.21.15.78:0/2903294024 <== mon.2 v2:172.21.15.78:3302/0 4 ==== mgrmap(e 5) v1 ==== 55212+0+0 (secure 0 0 0) 0x7f6a8c0107f0 con 0x7f6a9c071c80 2024-09-21T09:21:01.000 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.993+0000 7f517bfff640 1 -- 172.21.15.78:0/2487486260 <== mon.2 v2:172.21.15.78:3302/0 5 ==== osd_map(17..17 src has 1..17) v4 ==== 2868+0+0 (secure 0 0 0) 0x7f5188014070 con 0x7f5194097130 2024-09-21T09:21:01.001 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.993+0000 7f6a8affd640 1 -- 172.21.15.78:0/2903294024 <== mon.2 v2:172.21.15.78:3302/0 5 ==== osd_map(17..17 src has 1..17) v4 ==== 2868+0+0 (secure 0 0 0) 0x7f6a8c014070 con 0x7f6a9c071c80 2024-09-21T09:21:01.001 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:00.997+0000 7f517bfff640 1 -- 172.21.15.78:0/2487486260 <== mon.2 v2:172.21.15.78:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+146310 (secure 0 0 0) 0x7f518803b100 con 0x7f5194097130 2024-09-21T09:21:01.001 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.000+0000 7f6a8affd640 1 -- 172.21.15.78:0/2903294024 <== mon.2 v2:172.21.15.78:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+146310 (secure 0 0 0) 0x7f6a8c087ba0 con 0x7f6a9c071c80 2024-09-21T09:21:01.027 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.025+0000 7ff2a81b5640 1 -- 172.21.15.78:0/2804500779 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7ff2a0128d70 msgr2=0x7ff2a0129150 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:21:01.027 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.025+0000 7ff2a81b5640 1 --2- 172.21.15.78:0/2804500779 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7ff2a0128d70 0x7ff2a0129150 secure :-1 s=READY pgs=39 cs=0 l=1 rev1=1 crypto rx=0x7ff29c009830 tx=0x7ff29c02a480 comp rx=0 tx=0).stop 2024-09-21T09:21:01.028 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.026+0000 7ff2a81b5640 1 -- 172.21.15.78:0/2804500779 shutdown_connections 2024-09-21T09:21:01.028 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.026+0000 7ff2a81b5640 1 --2- 172.21.15.78:0/2804500779 >> v2:172.21.15.78:3300/0 conn(0x7ff2a012c500 0x7ff2a0134190 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:01.028 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.026+0000 7ff2a81b5640 1 --2- 172.21.15.78:0/2804500779 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7ff2a0128d70 0x7ff2a0129150 unknown :-1 s=CLOSED pgs=39 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:01.028 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.026+0000 7ff2a81b5640 1 -- 172.21.15.78:0/2804500779 >> 172.21.15.78:0/2804500779 conn(0x7ff2a009c8a0 msgr2=0x7ff2a009ccb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:21:01.028 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.026+0000 7ff2a81b5640 1 -- 172.21.15.78:0/2804500779 shutdown_connections 2024-09-21T09:21:01.028 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.026+0000 7ff2a81b5640 1 -- 172.21.15.78:0/2804500779 wait complete. 2024-09-21T09:21:01.031 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.027+0000 7ff2a81b5640 1 Processor -- start 2024-09-21T09:21:01.031 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.027+0000 7ff2a81b5640 1 -- start start 2024-09-21T09:21:01.031 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.027+0000 7ff2a81b5640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7ff2a0070b80 0x7ff2a012fed0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:21:01.031 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.027+0000 7ff2a81b5640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7ff2a0128d70 0x7ff2a0130410 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:21:01.031 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.028+0000 7ff2a81b5640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7ff2a012c500 0x7ff2a0130950 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:21:01.031 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.028+0000 7ff2a81b5640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff2a0133110 con 0x7ff2a0070b80 2024-09-21T09:21:01.031 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.028+0000 7ff2a81b5640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff2a0132a10 con 0x7ff2a012c500 2024-09-21T09:21:01.031 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.028+0000 7ff2a81b5640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 v1 -- 0x7ff2a01334e0 con 0x7ff2a0128d70 2024-09-21T09:21:01.031 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.028+0000 7ff2a5f51640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7ff2a0070b80 0x7ff2a012fed0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:21:01.032 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.028+0000 7ff2a6752640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7ff2a012c500 0x7ff2a0130950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:21:01.032 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.028+0000 7ff2a6752640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7ff2a012c500 0x7ff2a0130950 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3301/0 says I am v2:172.21.15.78:33056/0 (socket says 172.21.15.78:33056) 2024-09-21T09:21:01.032 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.028+0000 7ff2a5750640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7ff2a0128d70 0x7ff2a0130410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:21:01.032 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.028+0000 7ff2a6752640 1 -- 172.21.15.78:0/860836295 learned_addr learned my addr 172.21.15.78:0/860836295 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:21:01.032 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.028+0000 7ff2a5f51640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7ff2a0070b80 0x7ff2a012fed0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3300/0 says I am v2:172.21.15.78:37746/0 (socket says 172.21.15.78:37746) 2024-09-21T09:21:01.032 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.028+0000 7ff2a5750640 1 -- 172.21.15.78:0/860836295 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7ff2a012c500 msgr2=0x7ff2a0130950 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:21:01.032 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.028+0000 7ff2a5750640 1 --2- 172.21.15.78:0/860836295 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7ff2a012c500 0x7ff2a0130950 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:01.032 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.028+0000 7ff2a5750640 1 -- 172.21.15.78:0/860836295 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7ff2a0070b80 msgr2=0x7ff2a012fed0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:21:01.032 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.028+0000 7ff2a5750640 1 --2- 172.21.15.78:0/860836295 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7ff2a0070b80 0x7ff2a012fed0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:01.032 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.028+0000 7ff2a5750640 1 -- 172.21.15.78:0/860836295 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff29c009490 con 0x7ff2a0128d70 2024-09-21T09:21:01.032 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.028+0000 7ff2a5f51640 1 --2- 172.21.15.78:0/860836295 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7ff2a0070b80 0x7ff2a012fed0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:21:01.032 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.028+0000 7ff2a5750640 1 --2- 172.21.15.78:0/860836295 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7ff2a0128d70 0x7ff2a0130410 secure :-1 s=READY pgs=40 cs=0 l=1 rev1=1 crypto rx=0x7ff29000c750 tx=0x7ff29000cc10 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:21:01.032 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.029+0000 7ff2967fc640 1 -- 172.21.15.78:0/860836295 <== mon.2 v2:172.21.15.78:3302/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff29000a640 con 0x7ff2a0128d70 2024-09-21T09:21:01.033 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.029+0000 7ff2967fc640 1 -- 172.21.15.78:0/860836295 <== mon.2 v2:172.21.15.78:3302/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7ff29000a890 con 0x7ff2a0128d70 2024-09-21T09:21:01.033 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.029+0000 7ff2967fc640 1 -- 172.21.15.78:0/860836295 <== mon.2 v2:172.21.15.78:3302/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff29000ace0 con 0x7ff2a0128d70 2024-09-21T09:21:01.033 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.029+0000 7ff2a81b5640 1 -- 172.21.15.78:0/860836295 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff2a01320a0 con 0x7ff2a0128d70 2024-09-21T09:21:01.033 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.029+0000 7ff2a81b5640 1 -- 172.21.15.78:0/860836295 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff2a0134240 con 0x7ff2a0128d70 2024-09-21T09:21:01.033 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.030+0000 7ff2a81b5640 1 -- 172.21.15.78:0/860836295 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff2a0134520 con 0x7ff2a0128d70 2024-09-21T09:21:01.033 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.030+0000 7ff2967fc640 1 -- 172.21.15.78:0/860836295 <== mon.2 v2:172.21.15.78:3302/0 4 ==== mgrmap(e 5) v1 ==== 55212+0+0 (secure 0 0 0) 0x7ff290012420 con 0x7ff2a0128d70 2024-09-21T09:21:01.033 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.031+0000 7ff2967fc640 1 -- 172.21.15.78:0/860836295 <== mon.2 v2:172.21.15.78:3302/0 5 ==== osd_map(17..17 src has 1..17) v4 ==== 2868+0+0 (secure 0 0 0) 0x7ff29005c8c0 con 0x7ff2a0128d70 2024-09-21T09:21:01.035 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.034+0000 7ff2967fc640 1 -- 172.21.15.78:0/860836295 <== mon.2 v2:172.21.15.78:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+146310 (secure 0 0 0) 0x7ff29003a4d0 con 0x7ff2a0128d70 2024-09-21T09:21:01.040 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.038+0000 7fbba2fdb640 1 -- 172.21.15.78:0/2960532101 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fbb9c12bed0 msgr2=0x7fbb9c12e2c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:21:01.040 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.038+0000 7fbba2fdb640 1 --2- 172.21.15.78:0/2960532101 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fbb9c12bed0 0x7fbb9c12e2c0 secure :-1 s=READY pgs=52 cs=0 l=1 rev1=1 crypto rx=0x7fbb88009830 tx=0x7fbb8802a3f0 comp rx=0 tx=0).stop 2024-09-21T09:21:01.040 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.038+0000 7fbba2fdb640 1 -- 172.21.15.78:0/2960532101 shutdown_connections 2024-09-21T09:21:01.040 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.038+0000 7fbba2fdb640 1 --2- 172.21.15.78:0/2960532101 >> v2:172.21.15.78:3300/0 conn(0x7fbb9c12e800 0x7fbb9c130bf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:01.040 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.038+0000 7fbba2fdb640 1 --2- 172.21.15.78:0/2960532101 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fbb9c12bed0 0x7fbb9c12e2c0 unknown :-1 s=CLOSED pgs=52 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:01.040 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.038+0000 7fbba2fdb640 1 -- 172.21.15.78:0/2960532101 >> 172.21.15.78:0/2960532101 conn(0x7fbb9c09b5a0 msgr2=0x7fbb9c09b9b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:21:01.040 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.038+0000 7fbba2fdb640 1 -- 172.21.15.78:0/2960532101 shutdown_connections 2024-09-21T09:21:01.040 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.038+0000 7fbba2fdb640 1 -- 172.21.15.78:0/2960532101 wait complete. 2024-09-21T09:21:01.041 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.040+0000 7fbba2fdb640 1 Processor -- start 2024-09-21T09:21:01.041 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.040+0000 7fbba2fdb640 1 -- start start 2024-09-21T09:21:01.043 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.040+0000 7fbba2fdb640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fbb9c1290e0 0x7fbb9c120540 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:21:01.043 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.040+0000 7fbba2fdb640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fbb9c12bed0 0x7fbb9c120a80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:21:01.043 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.040+0000 7fbba2fdb640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fbb9c12e800 0x7fbb9c120fc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:21:01.044 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.041+0000 7fbb9bfff640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fbb9c12bed0 0x7fbb9c120a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:21:01.044 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.041+0000 7fbb9bfff640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fbb9c12bed0 0x7fbb9c120a80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3301/0 says I am v2:172.21.15.78:33078/0 (socket says 172.21.15.78:33078) 2024-09-21T09:21:01.044 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.041+0000 7fbba0d77640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fbb9c1290e0 0x7fbb9c120540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:21:01.044 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.041+0000 7fbba1578640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fbb9c12e800 0x7fbb9c120fc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:21:01.044 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.041+0000 7fbba0d77640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fbb9c1290e0 0x7fbb9c120540 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3302/0 says I am v2:172.21.15.78:35984/0 (socket says 172.21.15.78:35984) 2024-09-21T09:21:01.044 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.041+0000 7fbba1578640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fbb9c12e800 0x7fbb9c120fc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3300/0 says I am v2:172.21.15.78:37762/0 (socket says 172.21.15.78:37762) 2024-09-21T09:21:01.044 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.041+0000 7fbba2fdb640 1 -- 172.21.15.78:0/1001520114 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbb9c123d90 con 0x7fbb9c12e800 2024-09-21T09:21:01.044 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.041+0000 7fbb9bfff640 1 -- 172.21.15.78:0/1001520114 learned_addr learned my addr 172.21.15.78:0/1001520114 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:21:01.044 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.041+0000 7fbba2fdb640 1 -- 172.21.15.78:0/1001520114 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fbb9c124160 con 0x7fbb9c12bed0 2024-09-21T09:21:01.044 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.041+0000 7fbba2fdb640 1 -- 172.21.15.78:0/1001520114 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 v1 -- 0x7fbb9c123890 con 0x7fbb9c1290e0 2024-09-21T09:21:01.044 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.041+0000 7fbb9bfff640 1 -- 172.21.15.78:0/1001520114 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fbb9c1290e0 msgr2=0x7fbb9c120540 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:21:01.044 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.041+0000 7fbb9bfff640 1 --2- 172.21.15.78:0/1001520114 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fbb9c1290e0 0x7fbb9c120540 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:01.045 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.041+0000 7fbb9bfff640 1 -- 172.21.15.78:0/1001520114 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fbb9c12e800 msgr2=0x7fbb9c120fc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:21:01.045 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.041+0000 7fbb9bfff640 1 --2- 172.21.15.78:0/1001520114 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fbb9c12e800 0x7fbb9c120fc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:01.045 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.041+0000 7fbb9bfff640 1 -- 172.21.15.78:0/1001520114 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbb88009490 con 0x7fbb9c12bed0 2024-09-21T09:21:01.045 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.041+0000 7fbba0d77640 1 --2- 172.21.15.78:0/1001520114 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fbb9c1290e0 0x7fbb9c120540 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-09-21T09:21:01.045 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.041+0000 7fbba1578640 1 --2- 172.21.15.78:0/1001520114 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fbb9c12e800 0x7fbb9c120fc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:21:01.045 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.041+0000 7fbb9bfff640 1 --2- 172.21.15.78:0/1001520114 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fbb9c12bed0 0x7fbb9c120a80 secure :-1 s=READY pgs=53 cs=0 l=1 rev1=1 crypto rx=0x7fbb88000c00 tx=0x7fbb88003b90 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:21:01.045 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.042+0000 7fbb997fa640 1 -- 172.21.15.78:0/1001520114 <== mon.1 v2:172.21.15.78:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbb8803c620 con 0x7fbb9c12bed0 2024-09-21T09:21:01.045 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.042+0000 7fbb997fa640 1 -- 172.21.15.78:0/1001520114 <== mon.1 v2:172.21.15.78:3301/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7fbb8803c870 con 0x7fbb9c12bed0 2024-09-21T09:21:01.045 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.042+0000 7fbba2fdb640 1 -- 172.21.15.78:0/1001520114 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fbb9c1232b0 con 0x7fbb9c12bed0 2024-09-21T09:21:01.045 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.042+0000 7fbb997fa640 1 -- 172.21.15.78:0/1001520114 <== mon.1 v2:172.21.15.78:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbb8803cc80 con 0x7fbb9c12bed0 2024-09-21T09:21:01.045 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.042+0000 7fbba2fdb640 1 -- 172.21.15.78:0/1001520114 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fbb9c1bf700 con 0x7fbb9c12bed0 2024-09-21T09:21:01.046 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.043+0000 7fbba2fdb640 1 -- 172.21.15.78:0/1001520114 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fbb9c1294c0 con 0x7fbb9c12bed0 2024-09-21T09:21:01.046 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.043+0000 7fbb997fa640 1 -- 172.21.15.78:0/1001520114 <== mon.1 v2:172.21.15.78:3301/0 4 ==== mgrmap(e 5) v1 ==== 55212+0+0 (secure 0 0 0) 0x7fbb88044400 con 0x7fbb9c12bed0 2024-09-21T09:21:01.046 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.044+0000 7fbb997fa640 1 -- 172.21.15.78:0/1001520114 <== mon.1 v2:172.21.15.78:3301/0 5 ==== osd_map(17..17 src has 1..17) v4 ==== 2868+0+0 (secure 0 0 0) 0x7fbb8807df30 con 0x7fbb9c12bed0 2024-09-21T09:21:01.052 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.051+0000 7fbb997fa640 1 -- 172.21.15.78:0/1001520114 <== mon.1 v2:172.21.15.78:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+146310 (secure 0 0 0) 0x7fbb8803cde0 con 0x7fbb9c12bed0 2024-09-21T09:21:01.146 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.145+0000 7f6aa08e5640 1 -- 172.21.15.78:0/2903294024 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 2} v 0) v1 -- 0x7f6a9c08ede0 con 0x7f6a9c071c80 2024-09-21T09:21:01.146 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.145+0000 7f6a8affd640 1 -- 172.21.15.78:0/2903294024 <== mon.2 v2:172.21.15.78:3302/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 2}]=0 v0) v1 ==== 74+0+12 (secure 0 0 0) 0x7f6a8c03ae60 con 0x7f6a9c071c80 2024-09-21T09:21:01.147 INFO:teuthology.orchestra.run.smithi078.stdout:38654705669 2024-09-21T09:21:01.150 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.148+0000 7f6aa08e5640 1 -- 172.21.15.78:0/2903294024 >> v1:172.21.15.78:6816/2602406893 conn(0x7f6a7c047590 legacy=0x7f6a7c049a50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:21:01.150 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.148+0000 7f6aa08e5640 1 -- 172.21.15.78:0/2903294024 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f6a9c071c80 msgr2=0x7f6a9c133520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:21:01.150 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.148+0000 7f6aa08e5640 1 --2- 172.21.15.78:0/2903294024 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f6a9c071c80 0x7f6a9c133520 secure :-1 s=READY pgs=38 cs=0 l=1 rev1=1 crypto rx=0x7f6a8c00e790 tx=0x7f6a8c00ec50 comp rx=0 tx=0).stop 2024-09-21T09:21:01.150 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.148+0000 7f6aa08e5640 1 -- 172.21.15.78:0/2903294024 shutdown_connections 2024-09-21T09:21:01.151 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.148+0000 7f6aa08e5640 1 --2- 172.21.15.78:0/2903294024 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f6a9c097550 0x7f6a9c133fa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:01.151 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.148+0000 7f6aa08e5640 1 --2- 172.21.15.78:0/2903294024 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f6a9c096c30 0x7f6a9c133a60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:01.151 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.148+0000 7f6aa08e5640 1 --2- 172.21.15.78:0/2903294024 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f6a9c071c80 0x7f6a9c133520 unknown :-1 s=CLOSED pgs=38 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:01.151 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.148+0000 7f6aa08e5640 1 -- 172.21.15.78:0/2903294024 >> 172.21.15.78:0/2903294024 conn(0x7f6a9c09ed50 msgr2=0x7f6a9c1257f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:21:01.151 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.148+0000 7f6aa08e5640 1 -- 172.21.15.78:0/2903294024 shutdown_connections 2024-09-21T09:21:01.151 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.148+0000 7f6aa08e5640 1 -- 172.21.15.78:0/2903294024 wait complete. 2024-09-21T09:21:01.164 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705670 got 38654705669 for osd.2 2024-09-21T09:21:01.170 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.169+0000 7f519989c640 1 -- 172.21.15.78:0/2487486260 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 1} v 0) v1 -- 0x7f5194091460 con 0x7f5194097130 2024-09-21T09:21:01.170 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.169+0000 7f517bfff640 1 -- 172.21.15.78:0/2487486260 <== mon.2 v2:172.21.15.78:3302/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 1}]=0 v0) v1 ==== 74+0+12 (secure 0 0 0) 0x7f518803e720 con 0x7f5194097130 2024-09-21T09:21:01.171 INFO:teuthology.orchestra.run.smithi078.stdout:38654705667 2024-09-21T09:21:01.172 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.171+0000 7f519989c640 1 -- 172.21.15.78:0/2487486260 >> v1:172.21.15.78:6816/2602406893 conn(0x7f5180047560 legacy=0x7f5180049a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:21:01.173 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.171+0000 7f519989c640 1 -- 172.21.15.78:0/2487486260 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f5194097130 msgr2=0x7f519406d060 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:21:01.173 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.171+0000 7f519989c640 1 --2- 172.21.15.78:0/2487486260 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f5194097130 0x7f519406d060 secure :-1 s=READY pgs=37 cs=0 l=1 rev1=1 crypto rx=0x7f518800e980 tx=0x7f518800ee40 comp rx=0 tx=0).stop 2024-09-21T09:21:01.173 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.171+0000 7f519989c640 1 -- 172.21.15.78:0/2487486260 shutdown_connections 2024-09-21T09:21:01.173 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.171+0000 7f519989c640 1 --2- 172.21.15.78:0/2487486260 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f5194129e40 0x7f519406d5a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:01.173 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.171+0000 7f519989c640 1 --2- 172.21.15.78:0/2487486260 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f5194097130 0x7f519406d060 unknown :-1 s=CLOSED pgs=37 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:01.173 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.171+0000 7f519989c640 1 --2- 172.21.15.78:0/2487486260 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f519406ed30 0x7f519406cb20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:01.173 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.171+0000 7f519989c640 1 -- 172.21.15.78:0/2487486260 >> 172.21.15.78:0/2487486260 conn(0x7f5194097c00 msgr2=0x7f51940995a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:21:01.173 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.171+0000 7f519989c640 1 -- 172.21.15.78:0/2487486260 shutdown_connections 2024-09-21T09:21:01.173 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.171+0000 7f519989c640 1 -- 172.21.15.78:0/2487486260 wait complete. 2024-09-21T09:21:01.183 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705668 got 38654705667 for osd.1 2024-09-21T09:21:01.183 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.181+0000 7ff2a81b5640 1 -- 172.21.15.78:0/860836295 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 0} v 0) v1 -- 0x7ff2a0129150 con 0x7ff2a0128d70 2024-09-21T09:21:01.184 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.182+0000 7ff2967fc640 1 -- 172.21.15.78:0/860836295 <== mon.2 v2:172.21.15.78:3302/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 0}]=0 v0) v1 ==== 74+0+12 (secure 0 0 0) 0x7ff290016070 con 0x7ff2a0128d70 2024-09-21T09:21:01.184 INFO:teuthology.orchestra.run.smithi078.stdout:38654705667 2024-09-21T09:21:01.185 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.184+0000 7ff2a81b5640 1 -- 172.21.15.78:0/860836295 >> v1:172.21.15.78:6816/2602406893 conn(0x7ff284047560 legacy=0x7ff284049a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:21:01.185 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.184+0000 7ff2a81b5640 1 -- 172.21.15.78:0/860836295 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7ff2a0128d70 msgr2=0x7ff2a0130410 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:21:01.185 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.184+0000 7ff2a81b5640 1 --2- 172.21.15.78:0/860836295 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7ff2a0128d70 0x7ff2a0130410 secure :-1 s=READY pgs=40 cs=0 l=1 rev1=1 crypto rx=0x7ff29000c750 tx=0x7ff29000cc10 comp rx=0 tx=0).stop 2024-09-21T09:21:01.185 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.184+0000 7ff2a81b5640 1 -- 172.21.15.78:0/860836295 shutdown_connections 2024-09-21T09:21:01.186 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.184+0000 7ff2a81b5640 1 --2- 172.21.15.78:0/860836295 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7ff2a012c500 0x7ff2a0130950 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:01.186 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.184+0000 7ff2a81b5640 1 --2- 172.21.15.78:0/860836295 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7ff2a0128d70 0x7ff2a0130410 unknown :-1 s=CLOSED pgs=40 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:01.186 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.184+0000 7ff2a81b5640 1 --2- 172.21.15.78:0/860836295 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7ff2a0070b80 0x7ff2a012fed0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:01.186 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.184+0000 7ff2a81b5640 1 -- 172.21.15.78:0/860836295 >> 172.21.15.78:0/860836295 conn(0x7ff2a009c8a0 msgr2=0x7ff2a0125270 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:21:01.186 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.184+0000 7ff2a81b5640 1 -- 172.21.15.78:0/860836295 shutdown_connections 2024-09-21T09:21:01.186 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.184+0000 7ff2a81b5640 1 -- 172.21.15.78:0/860836295 wait complete. 2024-09-21T09:21:01.199 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705668 got 38654705667 for osd.0 2024-09-21T09:21:01.203 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.201+0000 7fbba2fdb640 1 -- 172.21.15.78:0/1001520114 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 3} v 0) v1 -- 0x7fbb9c073460 con 0x7fbb9c12bed0 2024-09-21T09:21:01.203 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.201+0000 7fbb997fa640 1 -- 172.21.15.78:0/1001520114 <== mon.1 v2:172.21.15.78:3301/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 3}]=0 v0) v1 ==== 74+0+12 (secure 0 0 0) 0x7fbb88049030 con 0x7fbb9c12bed0 2024-09-21T09:21:01.203 INFO:teuthology.orchestra.run.smithi078.stdout:38654705667 2024-09-21T09:21:01.205 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.203+0000 7fbba2fdb640 1 -- 172.21.15.78:0/1001520114 >> v1:172.21.15.78:6816/2602406893 conn(0x7fbb80047560 legacy=0x7fbb80049a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:21:01.205 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.203+0000 7fbba2fdb640 1 -- 172.21.15.78:0/1001520114 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fbb9c12bed0 msgr2=0x7fbb9c120a80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:21:01.205 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.203+0000 7fbba2fdb640 1 --2- 172.21.15.78:0/1001520114 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fbb9c12bed0 0x7fbb9c120a80 secure :-1 s=READY pgs=53 cs=0 l=1 rev1=1 crypto rx=0x7fbb88000c00 tx=0x7fbb88003b90 comp rx=0 tx=0).stop 2024-09-21T09:21:01.205 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.203+0000 7fbba2fdb640 1 -- 172.21.15.78:0/1001520114 shutdown_connections 2024-09-21T09:21:01.205 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.203+0000 7fbba2fdb640 1 --2- 172.21.15.78:0/1001520114 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fbb9c12e800 0x7fbb9c120fc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:01.205 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.203+0000 7fbba2fdb640 1 --2- 172.21.15.78:0/1001520114 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fbb9c12bed0 0x7fbb9c120a80 unknown :-1 s=CLOSED pgs=53 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:01.205 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.203+0000 7fbba2fdb640 1 --2- 172.21.15.78:0/1001520114 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fbb9c1290e0 0x7fbb9c120540 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:01.205 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.203+0000 7fbba2fdb640 1 -- 172.21.15.78:0/1001520114 >> 172.21.15.78:0/1001520114 conn(0x7fbb9c09b5a0 msgr2=0x7fbb9c127210 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:21:01.206 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.203+0000 7fbba2fdb640 1 -- 172.21.15.78:0/1001520114 shutdown_connections 2024-09-21T09:21:01.206 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:01.203+0000 7fbba2fdb640 1 -- 172.21.15.78:0/1001520114 wait complete. 2024-09-21T09:21:01.215 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705668 got 38654705667 for osd.3 2024-09-21T09:21:02.165 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.2 2024-09-21T09:21:02.185 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.1 2024-09-21T09:21:02.200 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.0 2024-09-21T09:21:02.216 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.3 2024-09-21T09:21:02.279 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.277+0000 7f1d60e3a640 1 -- 172.21.15.78:0/4118336988 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f1d5c06dde0 msgr2=0x7f1d5c070430 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:21:02.279 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.277+0000 7f1d60e3a640 1 --2- 172.21.15.78:0/4118336988 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f1d5c06dde0 0x7f1d5c070430 secure :-1 s=READY pgs=41 cs=0 l=1 rev1=1 crypto rx=0x7f1d54005fd0 tx=0x7f1d5402a8e0 comp rx=0 tx=0).stop 2024-09-21T09:21:02.279 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.277+0000 7f1d60e3a640 1 -- 172.21.15.78:0/4118336988 shutdown_connections 2024-09-21T09:21:02.279 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.277+0000 7f1d60e3a640 1 --2- 172.21.15.78:0/4118336988 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f1d5c06dde0 0x7f1d5c070430 unknown :-1 s=CLOSED pgs=41 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:02.279 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.277+0000 7f1d60e3a640 1 --2- 172.21.15.78:0/4118336988 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f1d5c071420 0x7f1d5c06fef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:02.279 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.277+0000 7f1d60e3a640 1 --2- 172.21.15.78:0/4118336988 >> v2:172.21.15.78:3300/0 conn(0x7f1d5c06d4c0 0x7f1d5c06d8a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:02.279 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.277+0000 7f1d60e3a640 1 -- 172.21.15.78:0/4118336988 >> 172.21.15.78:0/4118336988 conn(0x7f1d5c098050 msgr2=0x7f1d5c098460 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:21:02.279 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.277+0000 7f1d60e3a640 1 -- 172.21.15.78:0/4118336988 shutdown_connections 2024-09-21T09:21:02.279 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.277+0000 7f1d60e3a640 1 -- 172.21.15.78:0/4118336988 wait complete. 2024-09-21T09:21:02.280 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.279+0000 7f1d60e3a640 1 Processor -- start 2024-09-21T09:21:02.280 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.279+0000 7f1d60e3a640 1 -- start start 2024-09-21T09:21:02.281 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.279+0000 7f1d60e3a640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f1d5c06d4c0 0x7f1d5c11ded0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:21:02.281 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.279+0000 7f1d60e3a640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f1d5c06dde0 0x7f1d5c11e410 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:21:02.281 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.279+0000 7f1d60e3a640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f1d5c071420 0x7f1d5c11e950 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:21:02.281 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.279+0000 7f1d60e3a640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1d5c12f430 con 0x7f1d5c06d4c0 2024-09-21T09:21:02.281 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.279+0000 7f1d60e3a640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1d5c120070 con 0x7f1d5c071420 2024-09-21T09:21:02.281 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.279+0000 7f1d60e3a640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 v1 -- 0x7f1d5c121340 con 0x7f1d5c06dde0 2024-09-21T09:21:02.281 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.280+0000 7f1d59d9b640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f1d5c06dde0 0x7f1d5c11e410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:21:02.281 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.280+0000 7f1d5a59c640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f1d5c06d4c0 0x7f1d5c11ded0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:21:02.281 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.280+0000 7f1d59d9b640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f1d5c06dde0 0x7f1d5c11e410 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3302/0 says I am v2:172.21.15.78:36012/0 (socket says 172.21.15.78:36012) 2024-09-21T09:21:02.281 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.280+0000 7f1d5a59c640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f1d5c06d4c0 0x7f1d5c11ded0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3300/0 says I am v2:172.21.15.78:37768/0 (socket says 172.21.15.78:37768) 2024-09-21T09:21:02.281 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.280+0000 7f1d59d9b640 1 -- 172.21.15.78:0/3609104217 learned_addr learned my addr 172.21.15.78:0/3609104217 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:21:02.283 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.280+0000 7f1d5a59c640 1 -- 172.21.15.78:0/3609104217 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f1d5c06dde0 msgr2=0x7f1d5c11e410 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:21:02.283 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.280+0000 7f1d5a59c640 1 --2- 172.21.15.78:0/3609104217 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f1d5c06dde0 0x7f1d5c11e410 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:02.283 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.280+0000 7f1d5a59c640 1 -- 172.21.15.78:0/3609104217 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f1d5c071420 msgr2=0x7f1d5c11e950 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-21T09:21:02.283 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.280+0000 7f1d5ad9d640 1 --2- 172.21.15.78:0/3609104217 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f1d5c071420 0x7f1d5c11e950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:21:02.283 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.280+0000 7f1d5a59c640 1 --2- 172.21.15.78:0/3609104217 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f1d5c071420 0x7f1d5c11e950 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:02.283 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.280+0000 7f1d5a59c640 1 -- 172.21.15.78:0/3609104217 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1d4c009490 con 0x7f1d5c06d4c0 2024-09-21T09:21:02.283 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.280+0000 7f1d59d9b640 1 --2- 172.21.15.78:0/3609104217 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f1d5c06dde0 0x7f1d5c11e410 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:21:02.283 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.280+0000 7f1d5ad9d640 1 --2- 172.21.15.78:0/3609104217 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f1d5c071420 0x7f1d5c11e950 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-09-21T09:21:02.283 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.280+0000 7f1d5a59c640 1 --2- 172.21.15.78:0/3609104217 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f1d5c06d4c0 0x7f1d5c11ded0 secure :-1 s=READY pgs=74 cs=0 l=1 rev1=1 crypto rx=0x7f1d4c00ea20 tx=0x7f1d4c00eee0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:21:02.284 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.281+0000 7f1d4affd640 1 -- 172.21.15.78:0/3609104217 <== mon.0 v2:172.21.15.78:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1d4c00cd50 con 0x7f1d5c06d4c0 2024-09-21T09:21:02.284 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.281+0000 7f1d4affd640 1 -- 172.21.15.78:0/3609104217 <== mon.0 v2:172.21.15.78:3300/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7f1d4c00ceb0 con 0x7f1d5c06d4c0 2024-09-21T09:21:02.284 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.281+0000 7f1d60e3a640 1 -- 172.21.15.78:0/3609104217 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1d54009900 con 0x7f1d5c06d4c0 2024-09-21T09:21:02.284 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.281+0000 7f1d60e3a640 1 -- 172.21.15.78:0/3609104217 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1d5c11f640 con 0x7f1d5c06d4c0 2024-09-21T09:21:02.284 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.282+0000 7f1d4affd640 1 -- 172.21.15.78:0/3609104217 <== mon.0 v2:172.21.15.78:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1d4c004280 con 0x7f1d5c06d4c0 2024-09-21T09:21:02.284 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.282+0000 7f1d60e3a640 1 -- 172.21.15.78:0/3609104217 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1d5c06d8a0 con 0x7f1d5c06d4c0 2024-09-21T09:21:02.284 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.283+0000 7f1d4affd640 1 -- 172.21.15.78:0/3609104217 <== mon.0 v2:172.21.15.78:3300/0 4 ==== mgrmap(e 5) v1 ==== 55212+0+0 (secure 0 0 0) 0x7f1d4c0035f0 con 0x7f1d5c06d4c0 2024-09-21T09:21:02.285 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.283+0000 7f1d4affd640 1 -- 172.21.15.78:0/3609104217 <== mon.0 v2:172.21.15.78:3300/0 5 ==== osd_map(17..17 src has 1..17) v4 ==== 2868+0+0 (secure 0 0 0) 0x7f1d4c014070 con 0x7f1d5c06d4c0 2024-09-21T09:21:02.291 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.290+0000 7f1d4affd640 1 -- 172.21.15.78:0/3609104217 <== mon.0 v2:172.21.15.78:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+146310 (secure 0 0 0) 0x7f1d4c087ba0 con 0x7f1d5c06d4c0 2024-09-21T09:21:02.319 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.317+0000 7fb29cfa0640 1 -- 172.21.15.78:0/724885031 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fb29806c100 msgr2=0x7fb29806c540 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:21:02.319 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.317+0000 7fb29cfa0640 1 --2- 172.21.15.78:0/724885031 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fb29806c100 0x7fb29806c540 secure :-1 s=READY pgs=76 cs=0 l=1 rev1=1 crypto rx=0x7fb28c009830 tx=0x7fb28c02a400 comp rx=0 tx=0).stop 2024-09-21T09:21:02.319 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.317+0000 7fb29cfa0640 1 -- 172.21.15.78:0/724885031 shutdown_connections 2024-09-21T09:21:02.319 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.317+0000 7fb29cfa0640 1 --2- 172.21.15.78:0/724885031 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fb29806c100 0x7fb29806c540 unknown :-1 s=CLOSED pgs=76 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:02.319 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.317+0000 7fb29cfa0640 1 --2- 172.21.15.78:0/724885031 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fb298098430 0x7fb29806bbc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:02.319 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.317+0000 7fb29cfa0640 1 -- 172.21.15.78:0/724885031 >> 172.21.15.78:0/724885031 conn(0x7fb29809b850 msgr2=0x7fb298097d20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:21:02.319 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.318+0000 7fb29cfa0640 1 -- 172.21.15.78:0/724885031 shutdown_connections 2024-09-21T09:21:02.319 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.318+0000 7fb29cfa0640 1 -- 172.21.15.78:0/724885031 wait complete. 2024-09-21T09:21:02.320 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.319+0000 7fb29cfa0640 1 Processor -- start 2024-09-21T09:21:02.320 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.319+0000 7fb29cfa0640 1 -- start start 2024-09-21T09:21:02.322 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.319+0000 7fb29cfa0640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fb29806c100 0x7fb29808fe60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:21:02.322 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.319+0000 7fb29cfa0640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fb298098430 0x7fb2980903a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:21:02.322 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.319+0000 7fb2977fe640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fb29806c100 0x7fb29808fe60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:21:02.322 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.319+0000 7fb2977fe640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fb29806c100 0x7fb29808fe60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3301/0 says I am v2:172.21.15.78:58204/0 (socket says 172.21.15.78:58204) 2024-09-21T09:21:02.322 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.319+0000 7fb29cfa0640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fb29812e3a0 0x7fb2980908e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:21:02.322 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.319+0000 7fb29cfa0640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb2980917e0 con 0x7fb298098430 2024-09-21T09:21:02.322 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.319+0000 7fb296ffd640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fb298098430 0x7fb2980903a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:21:02.322 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.319+0000 7fb29cfa0640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb298071630 con 0x7fb29806c100 2024-09-21T09:21:02.322 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.319+0000 7fb29cfa0640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 v1 -- 0x7fb298120400 con 0x7fb29812e3a0 2024-09-21T09:21:02.322 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.320+0000 7fb2977fe640 1 -- 172.21.15.78:0/1915342824 learned_addr learned my addr 172.21.15.78:0/1915342824 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:21:02.322 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.320+0000 7fb296ffd640 1 -- 172.21.15.78:0/1915342824 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fb29812e3a0 msgr2=0x7fb2980908e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:21:02.322 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.320+0000 7fb297fff640 1 --2- 172.21.15.78:0/1915342824 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fb29812e3a0 0x7fb2980908e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:21:02.323 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.320+0000 7fb296ffd640 1 --2- 172.21.15.78:0/1915342824 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fb29812e3a0 0x7fb2980908e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:02.323 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.320+0000 7fb296ffd640 1 -- 172.21.15.78:0/1915342824 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fb29806c100 msgr2=0x7fb29808fe60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:21:02.323 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.320+0000 7fb296ffd640 1 --2- 172.21.15.78:0/1915342824 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fb29806c100 0x7fb29808fe60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:02.323 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.320+0000 7fb296ffd640 1 -- 172.21.15.78:0/1915342824 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb28c009490 con 0x7fb298098430 2024-09-21T09:21:02.323 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.320+0000 7fb297fff640 1 --2- 172.21.15.78:0/1915342824 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fb29812e3a0 0x7fb2980908e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-09-21T09:21:02.323 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.320+0000 7fb2977fe640 1 --2- 172.21.15.78:0/1915342824 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fb29806c100 0x7fb29808fe60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:21:02.323 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.320+0000 7fb296ffd640 1 --2- 172.21.15.78:0/1915342824 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fb298098430 0x7fb2980903a0 secure :-1 s=READY pgs=78 cs=0 l=1 rev1=1 crypto rx=0x7fb28c031000 tx=0x7fb28c003bf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:21:02.323 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.320+0000 7fb27ffff640 1 -- 172.21.15.78:0/1915342824 <== mon.0 v2:172.21.15.78:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb28c005220 con 0x7fb298098430 2024-09-21T09:21:02.323 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.320+0000 7fab201ed640 1 -- 172.21.15.78:0/1934778307 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fab18093e50 msgr2=0x7fab1812e070 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:21:02.324 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.320+0000 7fab201ed640 1 --2- 172.21.15.78:0/1934778307 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fab18093e50 0x7fab1812e070 secure :-1 s=READY pgs=54 cs=0 l=1 rev1=1 crypto rx=0x7fab08009860 tx=0x7fab0802a3d0 comp rx=0 tx=0).stop 2024-09-21T09:21:02.324 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.320+0000 7fb27ffff640 1 -- 172.21.15.78:0/1915342824 <== mon.0 v2:172.21.15.78:3300/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7fb28c005490 con 0x7fb298098430 2024-09-21T09:21:02.324 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.320+0000 7fb29cfa0640 1 -- 172.21.15.78:0/1915342824 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb298090e20 con 0x7fb298098430 2024-09-21T09:21:02.324 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.321+0000 7fb29cfa0640 1 -- 172.21.15.78:0/1915342824 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb298120b10 con 0x7fb298098430 2024-09-21T09:21:02.325 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.321+0000 7fab201ed640 1 -- 172.21.15.78:0/1934778307 shutdown_connections 2024-09-21T09:21:02.325 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.321+0000 7fab201ed640 1 --2- 172.21.15.78:0/1934778307 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fab18093e50 0x7fab1812e070 unknown :-1 s=CLOSED pgs=54 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:02.325 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.321+0000 7fab201ed640 1 --2- 172.21.15.78:0/1934778307 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fab1812e5b0 0x7fab181309a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:02.325 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.321+0000 7fab201ed640 1 -- 172.21.15.78:0/1934778307 >> 172.21.15.78:0/1934778307 conn(0x7fab18097c00 msgr2=0x7fab18098010 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:21:02.325 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.321+0000 7fab201ed640 1 -- 172.21.15.78:0/1934778307 shutdown_connections 2024-09-21T09:21:02.325 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.321+0000 7fab201ed640 1 -- 172.21.15.78:0/1934778307 wait complete. 2024-09-21T09:21:02.325 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.321+0000 7fb27ffff640 1 -- 172.21.15.78:0/1915342824 <== mon.0 v2:172.21.15.78:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb28c004a00 con 0x7fb298098430 2024-09-21T09:21:02.326 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.321+0000 7fab201ed640 1 Processor -- start 2024-09-21T09:21:02.326 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.322+0000 7fab201ed640 1 -- start start 2024-09-21T09:21:02.326 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.322+0000 7fb27ffff640 1 -- 172.21.15.78:0/1915342824 <== mon.0 v2:172.21.15.78:3300/0 4 ==== mgrmap(e 5) v1 ==== 55212+0+0 (secure 0 0 0) 0x7fb28c004b60 con 0x7fb298098430 2024-09-21T09:21:02.326 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.322+0000 7fab201ed640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fab18093360 0x7fab18088fb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:21:02.326 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.322+0000 7fab201ed640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fab18093e50 0x7fab180894f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:21:02.327 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.322+0000 7fab201ed640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fab1812e5b0 0x7fab18089a30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:21:02.327 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.322+0000 7fab201ed640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fab1808b3e0 con 0x7fab18093e50 2024-09-21T09:21:02.327 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.322+0000 7fab201ed640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fab18071410 con 0x7fab18093360 2024-09-21T09:21:02.327 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.322+0000 7fab201ed640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 v1 -- 0x7fab1808c640 con 0x7fab1812e5b0 2024-09-21T09:21:02.327 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.322+0000 7fab1d788640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fab18093e50 0x7fab180894f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:21:02.327 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.322+0000 7fab1d788640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fab18093e50 0x7fab180894f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3300/0 says I am v2:172.21.15.78:37026/0 (socket says 172.21.15.78:37026) 2024-09-21T09:21:02.327 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.322+0000 7fab1d788640 1 -- 172.21.15.78:0/1557005892 learned_addr learned my addr 172.21.15.78:0/1557005892 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:21:02.327 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.322+0000 7fab1e78a640 1 --2- 172.21.15.78:0/1557005892 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fab1812e5b0 0x7fab18089a30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:21:02.327 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.322+0000 7fab1df89640 1 --2- 172.21.15.78:0/1557005892 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fab18093360 0x7fab18088fb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:21:02.327 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.322+0000 7fb29cfa0640 1 -- 172.21.15.78:0/1915342824 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb29806c540 con 0x7fb298098430 2024-09-21T09:21:02.328 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.323+0000 7fab1e78a640 1 -- 172.21.15.78:0/1557005892 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fab18093360 msgr2=0x7fab18088fb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:21:02.328 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.323+0000 7fab1e78a640 1 --2- 172.21.15.78:0/1557005892 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fab18093360 0x7fab18088fb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:02.328 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.323+0000 7fab1e78a640 1 -- 172.21.15.78:0/1557005892 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fab18093e50 msgr2=0x7fab180894f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:21:02.328 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.323+0000 7fab1e78a640 1 --2- 172.21.15.78:0/1557005892 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fab18093e50 0x7fab180894f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:02.328 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.323+0000 7fab1e78a640 1 -- 172.21.15.78:0/1557005892 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fab10009840 con 0x7fab1812e5b0 2024-09-21T09:21:02.328 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.323+0000 7fab1df89640 1 --2- 172.21.15.78:0/1557005892 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fab18093360 0x7fab18088fb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:21:02.328 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.323+0000 7fab1d788640 1 --2- 172.21.15.78:0/1557005892 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fab18093e50 0x7fab180894f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:21:02.329 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.323+0000 7fab1e78a640 1 --2- 172.21.15.78:0/1557005892 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fab1812e5b0 0x7fab18089a30 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7fab10005fd0 tx=0x7fab1000c6a0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:21:02.329 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.323+0000 7fab0e7fc640 1 -- 172.21.15.78:0/1557005892 <== mon.2 v2:172.21.15.78:3302/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fab10003b70 con 0x7fab1812e5b0 2024-09-21T09:21:02.329 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.323+0000 7fb27ffff640 1 -- 172.21.15.78:0/1915342824 <== mon.0 v2:172.21.15.78:3300/0 5 ==== osd_map(17..17 src has 1..17) v4 ==== 2868+0+0 (secure 0 0 0) 0x7fb28c07e040 con 0x7fb298098430 2024-09-21T09:21:02.330 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.323+0000 7fab0e7fc640 1 -- 172.21.15.78:0/1557005892 <== mon.2 v2:172.21.15.78:3302/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7fab10003cd0 con 0x7fab1812e5b0 2024-09-21T09:21:02.330 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.323+0000 7fab0e7fc640 1 -- 172.21.15.78:0/1557005892 <== mon.2 v2:172.21.15.78:3302/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fab10004280 con 0x7fab1812e5b0 2024-09-21T09:21:02.330 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.323+0000 7fab201ed640 1 -- 172.21.15.78:0/1557005892 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fab08009490 con 0x7fab1812e5b0 2024-09-21T09:21:02.330 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.323+0000 7fab201ed640 1 -- 172.21.15.78:0/1557005892 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fab1808aa00 con 0x7fab1812e5b0 2024-09-21T09:21:02.330 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.324+0000 7fab0e7fc640 1 -- 172.21.15.78:0/1557005892 <== mon.2 v2:172.21.15.78:3302/0 4 ==== mgrmap(e 5) v1 ==== 55212+0+0 (secure 0 0 0) 0x7fab100043e0 con 0x7fab1812e5b0 2024-09-21T09:21:02.330 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.325+0000 7fab0e7fc640 1 -- 172.21.15.78:0/1557005892 <== mon.2 v2:172.21.15.78:3302/0 5 ==== osd_map(17..17 src has 1..17) v4 ==== 2868+0+0 (secure 0 0 0) 0x7fab10014070 con 0x7fab1812e5b0 2024-09-21T09:21:02.330 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.325+0000 7fab201ed640 1 -- 172.21.15.78:0/1557005892 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fab18093740 con 0x7fab1812e5b0 2024-09-21T09:21:02.330 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.327+0000 7fb27ffff640 1 -- 172.21.15.78:0/1915342824 <== mon.0 v2:172.21.15.78:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+146310 (secure 0 0 0) 0x7fb28c046e40 con 0x7fb298098430 2024-09-21T09:21:02.333 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.332+0000 7fab0e7fc640 1 -- 172.21.15.78:0/1557005892 <== mon.2 v2:172.21.15.78:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+146310 (secure 0 0 0) 0x7fab1003b1c0 con 0x7fab1812e5b0 2024-09-21T09:21:02.363 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.361+0000 7f2aca918640 1 -- 172.21.15.78:0/2939263068 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f2ac412c500 msgr2=0x7f2ac4134190 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:21:02.363 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.361+0000 7f2aca918640 1 --2- 172.21.15.78:0/2939263068 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f2ac412c500 0x7f2ac4134190 secure :-1 s=READY pgs=80 cs=0 l=1 rev1=1 crypto rx=0x7f2abc005fd0 tx=0x7f2abc02a820 comp rx=0 tx=0).stop 2024-09-21T09:21:02.363 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.361+0000 7f2aca918640 1 -- 172.21.15.78:0/2939263068 shutdown_connections 2024-09-21T09:21:02.363 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.361+0000 7f2aca918640 1 --2- 172.21.15.78:0/2939263068 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f2ac412c500 0x7f2ac4134190 unknown :-1 s=CLOSED pgs=80 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:02.363 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.361+0000 7f2aca918640 1 --2- 172.21.15.78:0/2939263068 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f2ac4128d70 0x7f2ac4129150 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:02.363 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.361+0000 7f2aca918640 1 -- 172.21.15.78:0/2939263068 >> 172.21.15.78:0/2939263068 conn(0x7f2ac409c8a0 msgr2=0x7f2ac409ccb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:21:02.363 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.361+0000 7f2aca918640 1 -- 172.21.15.78:0/2939263068 shutdown_connections 2024-09-21T09:21:02.363 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.361+0000 7f2aca918640 1 -- 172.21.15.78:0/2939263068 wait complete. 2024-09-21T09:21:02.364 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.363+0000 7f2aca918640 1 Processor -- start 2024-09-21T09:21:02.364 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.363+0000 7f2aca918640 1 -- start start 2024-09-21T09:21:02.367 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.364+0000 7f2aca918640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f2ac4070b80 0x7f2ac4121820 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:21:02.367 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.364+0000 7f2aca918640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f2ac4128d70 0x7f2ac4121d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:21:02.367 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.364+0000 7f2ac3fff640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f2ac4070b80 0x7f2ac4121820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:21:02.367 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.364+0000 7f2ac3fff640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f2ac4070b80 0x7f2ac4121820 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3301/0 says I am v2:172.21.15.78:58224/0 (socket says 172.21.15.78:58224) 2024-09-21T09:21:02.367 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.364+0000 7f2ac3fff640 1 -- 172.21.15.78:0/382299397 learned_addr learned my addr 172.21.15.78:0/382299397 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:21:02.367 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.364+0000 7f2aca918640 1 --2- 172.21.15.78:0/382299397 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f2ac412c500 0x7f2ac41222a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:21:02.367 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.364+0000 7f2aca918640 1 -- 172.21.15.78:0/382299397 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2ac41265f0 con 0x7f2ac412c500 2024-09-21T09:21:02.367 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.364+0000 7f2ac37fe640 1 --2- 172.21.15.78:0/382299397 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f2ac4128d70 0x7f2ac4121d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:21:02.367 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.364+0000 7f2aca918640 1 -- 172.21.15.78:0/382299397 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2ac412cf90 con 0x7f2ac4070b80 2024-09-21T09:21:02.368 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.364+0000 7f2aca918640 1 -- 172.21.15.78:0/382299397 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 v1 -- 0x7f2ac412d7d0 con 0x7f2ac4128d70 2024-09-21T09:21:02.368 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.364+0000 7f2ac8eb5640 1 --2- 172.21.15.78:0/382299397 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f2ac412c500 0x7f2ac41222a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:21:02.368 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.365+0000 7f2ac3fff640 1 -- 172.21.15.78:0/382299397 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f2ac4128d70 msgr2=0x7f2ac4121d60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:21:02.368 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.365+0000 7f2ac3fff640 1 --2- 172.21.15.78:0/382299397 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f2ac4128d70 0x7f2ac4121d60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:02.368 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.365+0000 7f2ac3fff640 1 -- 172.21.15.78:0/382299397 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f2ac412c500 msgr2=0x7f2ac41222a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:21:02.368 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.365+0000 7f2ac3fff640 1 --2- 172.21.15.78:0/382299397 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f2ac412c500 0x7f2ac41222a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:02.368 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.365+0000 7f2ac3fff640 1 -- 172.21.15.78:0/382299397 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2abc009840 con 0x7f2ac4070b80 2024-09-21T09:21:02.368 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.365+0000 7f2ac8eb5640 1 --2- 172.21.15.78:0/382299397 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f2ac412c500 0x7f2ac41222a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:21:02.368 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.365+0000 7f2ac3fff640 1 --2- 172.21.15.78:0/382299397 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f2ac4070b80 0x7f2ac4121820 secure :-1 s=READY pgs=55 cs=0 l=1 rev1=1 crypto rx=0x7f2ab400e9c0 tx=0x7f2ab400ee80 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:21:02.368 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.365+0000 7f2ac0ff9640 1 -- 172.21.15.78:0/382299397 <== mon.1 v2:172.21.15.78:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2ab400ccc0 con 0x7f2ac4070b80 2024-09-21T09:21:02.368 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.366+0000 7f2ac0ff9640 1 -- 172.21.15.78:0/382299397 <== mon.1 v2:172.21.15.78:3301/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7f2ab400ce20 con 0x7f2ac4070b80 2024-09-21T09:21:02.368 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.366+0000 7f2ac0ff9640 1 -- 172.21.15.78:0/382299397 <== mon.1 v2:172.21.15.78:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2ab4005280 con 0x7f2ac4070b80 2024-09-21T09:21:02.368 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.366+0000 7f2aca918640 1 -- 172.21.15.78:0/382299397 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f2ac412ec10 con 0x7f2ac4070b80 2024-09-21T09:21:02.369 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.366+0000 7f2aca918640 1 -- 172.21.15.78:0/382299397 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2ac4133e00 con 0x7f2ac4070b80 2024-09-21T09:21:02.369 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.367+0000 7f2aca918640 1 -- 172.21.15.78:0/382299397 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f2ac40008d0 con 0x7f2ac4070b80 2024-09-21T09:21:02.369 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.367+0000 7f2ac0ff9640 1 -- 172.21.15.78:0/382299397 <== mon.1 v2:172.21.15.78:3301/0 4 ==== mgrmap(e 5) v1 ==== 55212+0+0 (secure 0 0 0) 0x7f2ab40053e0 con 0x7f2ac4070b80 2024-09-21T09:21:02.369 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.368+0000 7f2ac0ff9640 1 -- 172.21.15.78:0/382299397 <== mon.1 v2:172.21.15.78:3301/0 5 ==== osd_map(17..17 src has 1..17) v4 ==== 2868+0+0 (secure 0 0 0) 0x7f2ab4014070 con 0x7f2ac4070b80 2024-09-21T09:21:02.376 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.374+0000 7f2ac0ff9640 1 -- 172.21.15.78:0/382299397 <== mon.1 v2:172.21.15.78:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+146310 (secure 0 0 0) 0x7f2ab403ad70 con 0x7f2ac4070b80 2024-09-21T09:21:02.443 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.442+0000 7f1d60e3a640 1 -- 172.21.15.78:0/3609104217 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 2} v 0) v1 -- 0x7f1d5c120320 con 0x7f1d5c06d4c0 2024-09-21T09:21:02.443 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.442+0000 7f1d4affd640 1 -- 172.21.15.78:0/3609104217 <== mon.0 v2:172.21.15.78:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 2}]=0 v0) v1 ==== 74+0+12 (secure 0 0 0) 0x7f1d4c03ae80 con 0x7f1d5c06d4c0 2024-09-21T09:21:02.444 INFO:teuthology.orchestra.run.smithi078.stdout:38654705671 2024-09-21T09:21:02.446 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.444+0000 7f1d60e3a640 1 -- 172.21.15.78:0/3609104217 >> v1:172.21.15.78:6816/2602406893 conn(0x7f1d3c0475e0 legacy=0x7f1d3c049aa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:21:02.446 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.444+0000 7f1d60e3a640 1 -- 172.21.15.78:0/3609104217 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f1d5c06d4c0 msgr2=0x7f1d5c11ded0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:21:02.446 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.444+0000 7f1d60e3a640 1 --2- 172.21.15.78:0/3609104217 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f1d5c06d4c0 0x7f1d5c11ded0 secure :-1 s=READY pgs=74 cs=0 l=1 rev1=1 crypto rx=0x7f1d4c00ea20 tx=0x7f1d4c00eee0 comp rx=0 tx=0).stop 2024-09-21T09:21:02.446 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.445+0000 7f1d60e3a640 1 -- 172.21.15.78:0/3609104217 shutdown_connections 2024-09-21T09:21:02.446 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.445+0000 7f1d60e3a640 1 --2- 172.21.15.78:0/3609104217 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f1d5c071420 0x7f1d5c11e950 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:02.446 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.445+0000 7f1d60e3a640 1 --2- 172.21.15.78:0/3609104217 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f1d5c06dde0 0x7f1d5c11e410 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:02.446 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.445+0000 7f1d60e3a640 1 --2- 172.21.15.78:0/3609104217 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f1d5c06d4c0 0x7f1d5c11ded0 unknown :-1 s=CLOSED pgs=74 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:02.447 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.445+0000 7f1d60e3a640 1 -- 172.21.15.78:0/3609104217 >> 172.21.15.78:0/3609104217 conn(0x7f1d5c098050 msgr2=0x7f1d5c1316c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:21:02.447 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.445+0000 7f1d60e3a640 1 -- 172.21.15.78:0/3609104217 shutdown_connections 2024-09-21T09:21:02.447 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.445+0000 7f1d60e3a640 1 -- 172.21.15.78:0/3609104217 wait complete. 2024-09-21T09:21:02.460 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705670 got 38654705671 for osd.2 2024-09-21T09:21:02.461 DEBUG:teuthology.parallel:result is None 2024-09-21T09:21:02.480 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.479+0000 7fb29cfa0640 1 -- 172.21.15.78:0/1915342824 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 0} v 0) v1 -- 0x7fb29812fe90 con 0x7fb298098430 2024-09-21T09:21:02.480 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.479+0000 7fb27ffff640 1 -- 172.21.15.78:0/1915342824 <== mon.0 v2:172.21.15.78:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 0}]=0 v0) v1 ==== 74+0+12 (secure 0 0 0) 0x7fb28c05f780 con 0x7fb298098430 2024-09-21T09:21:02.481 INFO:teuthology.orchestra.run.smithi078.stdout:38654705669 2024-09-21T09:21:02.483 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.481+0000 7fb29cfa0640 1 -- 172.21.15.78:0/1915342824 >> v1:172.21.15.78:6816/2602406893 conn(0x7fb284047560 legacy=0x7fb284049a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:21:02.483 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.481+0000 7fb29cfa0640 1 -- 172.21.15.78:0/1915342824 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fb298098430 msgr2=0x7fb2980903a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:21:02.483 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.481+0000 7fb29cfa0640 1 --2- 172.21.15.78:0/1915342824 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fb298098430 0x7fb2980903a0 secure :-1 s=READY pgs=78 cs=0 l=1 rev1=1 crypto rx=0x7fb28c031000 tx=0x7fb28c003bf0 comp rx=0 tx=0).stop 2024-09-21T09:21:02.483 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.481+0000 7fb29cfa0640 1 -- 172.21.15.78:0/1915342824 shutdown_connections 2024-09-21T09:21:02.483 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.481+0000 7fb29cfa0640 1 --2- 172.21.15.78:0/1915342824 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fb29812e3a0 0x7fb2980908e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:02.483 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.481+0000 7fb29cfa0640 1 --2- 172.21.15.78:0/1915342824 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fb298098430 0x7fb2980903a0 unknown :-1 s=CLOSED pgs=78 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:02.483 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.481+0000 7fb29cfa0640 1 --2- 172.21.15.78:0/1915342824 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fb29806c100 0x7fb29808fe60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:02.483 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.481+0000 7fb29cfa0640 1 -- 172.21.15.78:0/1915342824 >> 172.21.15.78:0/1915342824 conn(0x7fb29809b850 msgr2=0x7fb2980700d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:21:02.483 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.482+0000 7fb29cfa0640 1 -- 172.21.15.78:0/1915342824 shutdown_connections 2024-09-21T09:21:02.484 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.482+0000 7fb29cfa0640 1 -- 172.21.15.78:0/1915342824 wait complete. 2024-09-21T09:21:02.489 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.488+0000 7fab201ed640 1 -- 172.21.15.78:0/1557005892 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 1} v 0) v1 -- 0x7fab18073460 con 0x7fab1812e5b0 2024-09-21T09:21:02.489 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.488+0000 7fab0e7fc640 1 -- 172.21.15.78:0/1557005892 <== mon.2 v2:172.21.15.78:3302/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 1}]=0 v0) v1 ==== 74+0+12 (secure 0 0 0) 0x7fab10003e80 con 0x7fab1812e5b0 2024-09-21T09:21:02.489 INFO:teuthology.orchestra.run.smithi078.stdout:38654705669 2024-09-21T09:21:02.492 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.490+0000 7fab201ed640 1 -- 172.21.15.78:0/1557005892 >> v1:172.21.15.78:6816/2602406893 conn(0x7faafc047560 legacy=0x7faafc049a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:21:02.492 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.490+0000 7fab201ed640 1 -- 172.21.15.78:0/1557005892 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fab1812e5b0 msgr2=0x7fab18089a30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:21:02.492 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.490+0000 7fab201ed640 1 --2- 172.21.15.78:0/1557005892 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fab1812e5b0 0x7fab18089a30 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7fab10005fd0 tx=0x7fab1000c6a0 comp rx=0 tx=0).stop 2024-09-21T09:21:02.492 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.490+0000 7fab201ed640 1 -- 172.21.15.78:0/1557005892 shutdown_connections 2024-09-21T09:21:02.492 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.490+0000 7fab201ed640 1 --2- 172.21.15.78:0/1557005892 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fab1812e5b0 0x7fab18089a30 unknown :-1 s=CLOSED pgs=42 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:02.492 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.490+0000 7fab201ed640 1 --2- 172.21.15.78:0/1557005892 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fab18093e50 0x7fab180894f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:02.492 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.491+0000 7fab201ed640 1 --2- 172.21.15.78:0/1557005892 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fab18093360 0x7fab18088fb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:02.493 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.491+0000 7fab201ed640 1 -- 172.21.15.78:0/1557005892 >> 172.21.15.78:0/1557005892 conn(0x7fab18097c00 msgr2=0x7fab180998e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:21:02.493 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.491+0000 7fab201ed640 1 -- 172.21.15.78:0/1557005892 shutdown_connections 2024-09-21T09:21:02.493 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.491+0000 7fab201ed640 1 -- 172.21.15.78:0/1557005892 wait complete. 2024-09-21T09:21:02.495 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705668 got 38654705669 for osd.0 2024-09-21T09:21:02.495 DEBUG:teuthology.parallel:result is None 2024-09-21T09:21:02.504 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705668 got 38654705669 for osd.1 2024-09-21T09:21:02.504 DEBUG:teuthology.parallel:result is None 2024-09-21T09:21:02.520 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.519+0000 7f2aca918640 1 -- 172.21.15.78:0/382299397 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 3} v 0) v1 -- 0x7f2ac4134b80 con 0x7f2ac4070b80 2024-09-21T09:21:02.521 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.520+0000 7f2ac0ff9640 1 -- 172.21.15.78:0/382299397 <== mon.1 v2:172.21.15.78:3301/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 3}]=0 v0) v1 ==== 74+0+12 (secure 0 0 0) 0x7f2ab403e390 con 0x7f2ac4070b80 2024-09-21T09:21:02.521 INFO:teuthology.orchestra.run.smithi078.stdout:38654705669 2024-09-21T09:21:02.523 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.521+0000 7f2aca918640 1 -- 172.21.15.78:0/382299397 >> v1:172.21.15.78:6816/2602406893 conn(0x7f2ab0047590 legacy=0x7f2ab0049a50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:21:02.523 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.521+0000 7f2aca918640 1 -- 172.21.15.78:0/382299397 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f2ac4070b80 msgr2=0x7f2ac4121820 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:21:02.523 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.521+0000 7f2aca918640 1 --2- 172.21.15.78:0/382299397 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f2ac4070b80 0x7f2ac4121820 secure :-1 s=READY pgs=55 cs=0 l=1 rev1=1 crypto rx=0x7f2ab400e9c0 tx=0x7f2ab400ee80 comp rx=0 tx=0).stop 2024-09-21T09:21:02.523 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.521+0000 7f2aca918640 1 -- 172.21.15.78:0/382299397 shutdown_connections 2024-09-21T09:21:02.523 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.521+0000 7f2aca918640 1 --2- 172.21.15.78:0/382299397 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f2ac412c500 0x7f2ac41222a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:02.523 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.521+0000 7f2aca918640 1 --2- 172.21.15.78:0/382299397 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f2ac4128d70 0x7f2ac4121d60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:02.523 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.521+0000 7f2aca918640 1 --2- 172.21.15.78:0/382299397 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f2ac4070b80 0x7f2ac4121820 unknown :-1 s=CLOSED pgs=55 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:02.523 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.521+0000 7f2aca918640 1 -- 172.21.15.78:0/382299397 >> 172.21.15.78:0/382299397 conn(0x7f2ac409c8a0 msgr2=0x7f2ac4127620 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:21:02.523 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.521+0000 7f2aca918640 1 -- 172.21.15.78:0/382299397 shutdown_connections 2024-09-21T09:21:02.524 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.521+0000 7f2aca918640 1 -- 172.21.15.78:0/382299397 wait complete. 2024-09-21T09:21:02.533 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705668 got 38654705669 for osd.3 2024-09-21T09:21:02.533 DEBUG:teuthology.parallel:result is None 2024-09-21T09:21:02.533 INFO:tasks.ceph.ceph_manager.ceph:waiting for clean 2024-09-21T09:21:02.533 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-09-21T09:21:02.673 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.671+0000 7f3d4e9ec640 1 -- 172.21.15.78:0/2616365626 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f3d48095c90 msgr2=0x7f3d4806e1b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:21:02.673 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.671+0000 7f3d4e9ec640 1 --2- 172.21.15.78:0/2616365626 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f3d48095c90 0x7f3d4806e1b0 secure :-1 s=READY pgs=56 cs=0 l=1 rev1=1 crypto rx=0x7f3d3c009900 tx=0x7f3d3c02a4d0 comp rx=0 tx=0).stop 2024-09-21T09:21:02.673 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.671+0000 7f3d4e9ec640 1 -- 172.21.15.78:0/2616365626 shutdown_connections 2024-09-21T09:21:02.673 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.671+0000 7f3d4e9ec640 1 --2- 172.21.15.78:0/2616365626 >> v2:172.21.15.78:3300/0 conn(0x7f3d48070840 0x7f3d4806e6f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:02.673 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.671+0000 7f3d4e9ec640 1 --2- 172.21.15.78:0/2616365626 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f3d48095c90 0x7f3d4806e1b0 unknown :-1 s=CLOSED pgs=56 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:02.673 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.671+0000 7f3d4e9ec640 1 --2- 172.21.15.78:0/2616365626 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f3d48095370 0x7f3d48095750 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:02.673 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.671+0000 7f3d4e9ec640 1 -- 172.21.15.78:0/2616365626 >> 172.21.15.78:0/2616365626 conn(0x7f3d48097590 msgr2=0x7f3d480979a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:21:02.674 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.671+0000 7f3d4e9ec640 1 -- 172.21.15.78:0/2616365626 shutdown_connections 2024-09-21T09:21:02.674 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.671+0000 7f3d4e9ec640 1 -- 172.21.15.78:0/2616365626 wait complete. 2024-09-21T09:21:02.675 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.672+0000 7f3d4e9ec640 1 Processor -- start 2024-09-21T09:21:02.675 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.672+0000 7f3d4e9ec640 1 -- start start 2024-09-21T09:21:02.676 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.672+0000 7f3d4e9ec640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f3d48070840 0x7f3d48126470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:21:02.676 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.672+0000 7f3d4e9ec640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f3d48095370 0x7f3d481269b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:21:02.676 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.673+0000 7f3d4e9ec640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f3d48095c90 0x7f3d48126ef0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:21:02.676 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.673+0000 7f3d4e9ec640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3d48129160 con 0x7f3d48095370 2024-09-21T09:21:02.676 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.673+0000 7f3d4e9ec640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3d48128240 con 0x7f3d48070840 2024-09-21T09:21:02.676 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.673+0000 7f3d4e9ec640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 v1 -- 0x7f3d48128e80 con 0x7f3d48095c90 2024-09-21T09:21:02.676 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.673+0000 7f3d47fff640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f3d48070840 0x7f3d48126470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:21:02.676 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.673+0000 7f3d477fe640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f3d48095370 0x7f3d481269b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:21:02.676 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.673+0000 7f3d47fff640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f3d48070840 0x7f3d48126470 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3301/0 says I am v2:172.21.15.78:58244/0 (socket says 172.21.15.78:58244) 2024-09-21T09:21:02.676 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.673+0000 7f3d47fff640 1 -- 172.21.15.78:0/1030061945 learned_addr learned my addr 172.21.15.78:0/1030061945 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:21:02.676 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.673+0000 7f3d4cf89640 1 --2- 172.21.15.78:0/1030061945 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f3d48095c90 0x7f3d48126ef0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:21:02.676 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.673+0000 7f3d47fff640 1 -- 172.21.15.78:0/1030061945 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f3d48095c90 msgr2=0x7f3d48126ef0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:21:02.676 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.673+0000 7f3d47fff640 1 --2- 172.21.15.78:0/1030061945 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f3d48095c90 0x7f3d48126ef0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:02.677 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.673+0000 7f3d47fff640 1 -- 172.21.15.78:0/1030061945 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f3d48095370 msgr2=0x7f3d481269b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:21:02.677 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.673+0000 7f3d47fff640 1 --2- 172.21.15.78:0/1030061945 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f3d48095370 0x7f3d481269b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:02.677 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.673+0000 7f3d47fff640 1 -- 172.21.15.78:0/1030061945 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3d40009840 con 0x7f3d48070840 2024-09-21T09:21:02.677 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.673+0000 7f3d477fe640 1 --2- 172.21.15.78:0/1030061945 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f3d48095370 0x7f3d481269b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:21:02.677 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.673+0000 7f3d47fff640 1 --2- 172.21.15.78:0/1030061945 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f3d48070840 0x7f3d48126470 secure :-1 s=READY pgs=57 cs=0 l=1 rev1=1 crypto rx=0x7f3d3800eb20 tx=0x7f3d3800ef20 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:21:02.677 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.673+0000 7f3d44ff9640 1 -- 172.21.15.78:0/1030061945 <== mon.1 v2:172.21.15.78:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3d3800cd40 con 0x7f3d48070840 2024-09-21T09:21:02.677 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.673+0000 7f3d44ff9640 1 -- 172.21.15.78:0/1030061945 <== mon.1 v2:172.21.15.78:3301/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7f3d38004020 con 0x7f3d48070840 2024-09-21T09:21:02.677 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.673+0000 7f3d44ff9640 1 -- 172.21.15.78:0/1030061945 <== mon.1 v2:172.21.15.78:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3d38004430 con 0x7f3d48070840 2024-09-21T09:21:02.677 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.673+0000 7f3d4e9ec640 1 -- 172.21.15.78:0/1030061945 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3d3800a5a0 con 0x7f3d48070840 2024-09-21T09:21:02.677 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.673+0000 7f3d4e9ec640 1 -- 172.21.15.78:0/1030061945 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3d3c009560 con 0x7f3d48070840 2024-09-21T09:21:02.677 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.674+0000 7f3d44ff9640 1 -- 172.21.15.78:0/1030061945 <== mon.1 v2:172.21.15.78:3301/0 4 ==== mgrmap(e 5) v1 ==== 55212+0+0 (secure 0 0 0) 0x7f3d380035f0 con 0x7f3d48070840 2024-09-21T09:21:02.677 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.674+0000 7f3d4e9ec640 1 -- 172.21.15.78:0/1030061945 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3d48095750 con 0x7f3d48070840 2024-09-21T09:21:02.678 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.675+0000 7f3d44ff9640 1 -- 172.21.15.78:0/1030061945 <== mon.1 v2:172.21.15.78:3301/0 5 ==== osd_map(17..17 src has 1..17) v4 ==== 2868+0+0 (secure 0 0 0) 0x7f3d3805ccb0 con 0x7f3d48070840 2024-09-21T09:21:02.679 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.678+0000 7f3d44ff9640 1 -- 172.21.15.78:0/1030061945 <== mon.1 v2:172.21.15.78:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+146310 (secure 0 0 0) 0x7f3d3803a960 con 0x7f3d48070840 2024-09-21T09:21:02.811 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.810+0000 7f3d4e9ec640 1 -- 172.21.15.78:0/1030061945 --> v1:172.21.15.78:6816/2602406893 -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7f3d48127430 con 0x7f3d34047560 2024-09-21T09:21:02.812 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.811+0000 7f3d44ff9640 1 -- 172.21.15.78:0/1030061945 <== mgr.4106 v1:172.21.15.78:6816/2602406893 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+34942 (unknown 2965378022 0 2833961459) 0x7f3d48127430 con 0x7f3d34047560 2024-09-21T09:21:02.813 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:21:02.813 INFO:teuthology.orchestra.run.smithi078.stderr:dumped all 2024-09-21T09:21:02.814 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.813+0000 7f3d4e9ec640 1 -- 172.21.15.78:0/1030061945 >> v1:172.21.15.78:6816/2602406893 conn(0x7f3d34047560 legacy=0x7f3d34049a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:21:02.814 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.813+0000 7f3d4e9ec640 1 -- 172.21.15.78:0/1030061945 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f3d48070840 msgr2=0x7f3d48126470 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:21:02.814 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.813+0000 7f3d4e9ec640 1 --2- 172.21.15.78:0/1030061945 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f3d48070840 0x7f3d48126470 secure :-1 s=READY pgs=57 cs=0 l=1 rev1=1 crypto rx=0x7f3d3800eb20 tx=0x7f3d3800ef20 comp rx=0 tx=0).stop 2024-09-21T09:21:02.815 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.813+0000 7f3d4e9ec640 1 -- 172.21.15.78:0/1030061945 shutdown_connections 2024-09-21T09:21:02.815 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.813+0000 7f3d4e9ec640 1 --2- 172.21.15.78:0/1030061945 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f3d48095c90 0x7f3d48126ef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:02.815 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.813+0000 7f3d4e9ec640 1 --2- 172.21.15.78:0/1030061945 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f3d48095370 0x7f3d481269b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:02.815 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.813+0000 7f3d4e9ec640 1 --2- 172.21.15.78:0/1030061945 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f3d48070840 0x7f3d48126470 unknown :-1 s=CLOSED pgs=57 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:02.815 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.813+0000 7f3d4e9ec640 1 -- 172.21.15.78:0/1030061945 >> 172.21.15.78:0/1030061945 conn(0x7f3d48097590 msgr2=0x7f3d4811f730 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:21:02.815 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.813+0000 7f3d4e9ec640 1 -- 172.21.15.78:0/1030061945 shutdown_connections 2024-09-21T09:21:02.815 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.813+0000 7f3d4e9ec640 1 -- 172.21.15.78:0/1030061945 wait complete. 2024-09-21T09:21:02.825 INFO:teuthology.orchestra.run.smithi078.stdout:{"pg_ready":true,"pg_map":{"version":20,"stamp":"2024-09-21T09:21:01.141659+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":139,"num_write_kb":1896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100,"ondisk_log_size":100,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":15,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":377487360,"kb_used":109844,"kb_used_data":1744,"kb_used_omap":6,"kb_used_meta":108025,"kb_avail":377377516,"statfs":{"total":386547056640,"available":386434576384,"internally_reserved":0,"allocated":1785856,"data_stored":1345730,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6374,"internal_metadata":110618394},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"5.828556"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":15,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-09-21T09:20:58.335685+0000","last_change":"2024-09-21T09:20:58.335685+0000","last_active":"2024-09-21T09:20:58.335685+0000","last_peered":"2024-09-21T09:20:58.335685+0000","last_clean":"2024-09-21T09:20:58.335685+0000","last_became_active":"2024-09-21T09:20:56.322152+0000","last_became_peered":"2024-09-21T09:20:56.322152+0000","last_unstale":"2024-09-21T09:20:58.335685+0000","last_undegraded":"2024-09-21T09:20:58.335685+0000","last_fullsized":"2024-09-21T09:20:58.335685+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:19:22.863125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039605,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":15,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-09-21T09:20:58.335404+0000","last_change":"2024-09-21T09:20:58.335404+0000","last_active":"2024-09-21T09:20:58.335404+0000","last_peered":"2024-09-21T09:20:58.335404+0000","last_clean":"2024-09-21T09:20:58.335404+0000","last_became_active":"2024-09-21T09:20:56.320383+0000","last_became_peered":"2024-09-21T09:20:56.320383+0000","last_unstale":"2024-09-21T09:20:58.335404+0000","last_undegraded":"2024-09-21T09:20:58.335404+0000","last_fullsized":"2024-09-21T09:20:58.335404+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:36:03.357395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040851400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":15,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-09-21T09:20:58.335829+0000","last_change":"2024-09-21T09:20:58.335829+0000","last_active":"2024-09-21T09:20:58.335829+0000","last_peered":"2024-09-21T09:20:58.335829+0000","last_clean":"2024-09-21T09:20:58.335829+0000","last_became_active":"2024-09-21T09:20:56.322789+0000","last_became_peered":"2024-09-21T09:20:56.322789+0000","last_unstale":"2024-09-21T09:20:58.335829+0000","last_undegraded":"2024-09-21T09:20:58.335829+0000","last_fullsized":"2024-09-21T09:20:58.335829+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:16:32.684631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001005517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":15,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-09-21T09:20:58.335406+0000","last_change":"2024-09-21T09:20:58.335406+0000","last_active":"2024-09-21T09:20:58.335406+0000","last_peered":"2024-09-21T09:20:58.335406+0000","last_clean":"2024-09-21T09:20:58.335406+0000","last_became_active":"2024-09-21T09:20:56.324636+0000","last_became_peered":"2024-09-21T09:20:56.324636+0000","last_unstale":"2024-09-21T09:20:58.335406+0000","last_undegraded":"2024-09-21T09:20:58.335406+0000","last_fullsized":"2024-09-21T09:20:58.335406+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:19:22.863125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054560899999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"17'2","reported_seq":17,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-09-21T09:20:58.343773+0000","last_change":"2024-09-21T09:20:58.336768+0000","last_active":"2024-09-21T09:20:58.343773+0000","last_peered":"2024-09-21T09:20:58.343773+0000","last_clean":"2024-09-21T09:20:58.343773+0000","last_became_active":"2024-09-21T09:20:56.327106+0000","last_became_peered":"2024-09-21T09:20:56.327106+0000","last_unstale":"2024-09-21T09:20:58.343773+0000","last_undegraded":"2024-09-21T09:20:58.343773+0000","last_fullsized":"2024-09-21T09:20:58.343773+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:45:36.558638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011881540000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":15,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-09-21T09:20:58.479452+0000","last_change":"2024-09-21T09:20:58.479452+0000","last_active":"2024-09-21T09:20:58.479452+0000","last_peered":"2024-09-21T09:20:58.479452+0000","last_clean":"2024-09-21T09:20:58.479452+0000","last_became_active":"2024-09-21T09:20:56.324347+0000","last_became_peered":"2024-09-21T09:20:56.324347+0000","last_unstale":"2024-09-21T09:20:58.479452+0000","last_undegraded":"2024-09-21T09:20:58.479452+0000","last_fullsized":"2024-09-21T09:20:58.479452+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:43:56.319711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023051899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":15,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-09-21T09:20:58.335174+0000","last_change":"2024-09-21T09:20:58.335174+0000","last_active":"2024-09-21T09:20:58.335174+0000","last_peered":"2024-09-21T09:20:58.335174+0000","last_clean":"2024-09-21T09:20:58.335174+0000","last_became_active":"2024-09-21T09:20:56.322710+0000","last_became_peered":"2024-09-21T09:20:56.322710+0000","last_unstale":"2024-09-21T09:20:58.335174+0000","last_undegraded":"2024-09-21T09:20:58.335174+0000","last_fullsized":"2024-09-21T09:20:58.335174+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:03:03.617340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039429200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"15'1","reported_seq":16,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-09-21T09:20:58.336756+0000","last_change":"2024-09-21T09:20:58.336756+0000","last_active":"2024-09-21T09:20:58.336756+0000","last_peered":"2024-09-21T09:20:58.336756+0000","last_clean":"2024-09-21T09:20:58.336756+0000","last_became_active":"2024-09-21T09:20:56.321532+0000","last_became_peered":"2024-09-21T09:20:56.321532+0000","last_unstale":"2024-09-21T09:20:58.336756+0000","last_undegraded":"2024-09-21T09:20:58.336756+0000","last_fullsized":"2024-09-21T09:20:58.336756+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:57:55.907381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091827900000000004,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"12'97","reported_seq":141,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-21T09:20:57.328245+0000","last_change":"2024-09-21T09:20:52.294610+0000","last_active":"2024-09-21T09:20:57.328245+0000","last_peered":"2024-09-21T09:20:57.328245+0000","last_clean":"2024-09-21T09:20:57.328245+0000","last_became_active":"2024-09-21T09:20:52.293626+0000","last_became_peered":"2024-09-21T09:20:52.293626+0000","last_unstale":"2024-09-21T09:20:57.328245+0000","last_undegraded":"2024-09-21T09:20:57.328245+0000","last_fullsized":"2024-09-21T09:20:57.328245+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:51.278939+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:51.278939+0000","last_clean_scrub_stamp":"2024-09-21T09:20:51.278939+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:43:02.095040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":4},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97,"ondisk_log_size":97,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":3,"up_from":9,"seq":38654705669,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27172,"kb_used_data":148,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344668,"statfs":{"total":96636764160,"available":96608940032,"internally_reserved":0,"allocated":151552,"data_stored":41258,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":27654600},"hb_peers":[0,1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.38400000000000001}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45000000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46800000000000003}]}]},{"osd":2,"up_from":9,"seq":38654705671,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27168,"kb_used_data":144,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344672,"statfs":{"total":96636764160,"available":96608944128,"internally_reserved":0,"allocated":147456,"data_stored":41239,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.33600000000000002}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26200000000000001}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.248}]}]},{"osd":1,"up_from":9,"seq":38654705669,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27756,"kb_used_data":728,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344084,"statfs":{"total":96636764160,"available":96608342016,"internally_reserved":0,"allocated":745472,"data_stored":631626,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45200000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51500000000000001}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47299999999999998}]}]},{"osd":0,"up_from":9,"seq":38654705669,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27748,"kb_used_data":724,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344092,"statfs":{"total":96636764160,"available":96608350208,"internally_reserved":0,"allocated":741376,"data_stored":631607,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[1,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.255}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.37}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.33000000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T09:21:02.826 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-09-21T09:21:02.924 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.923+0000 7f9c9ab9c640 1 -- 172.21.15.78:0/2294944221 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f9c9406ff10 msgr2=0x7f9c9412ca30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:21:02.925 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.923+0000 7f9c9ab9c640 1 --2- 172.21.15.78:0/2294944221 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f9c9406ff10 0x7f9c9412ca30 secure :-1 s=READY pgs=58 cs=0 l=1 rev1=1 crypto rx=0x7f9c80009900 tx=0x7f9c8002a4a0 comp rx=0 tx=0).stop 2024-09-21T09:21:02.925 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.923+0000 7f9c9ab9c640 1 -- 172.21.15.78:0/2294944221 shutdown_connections 2024-09-21T09:21:02.925 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.923+0000 7f9c9ab9c640 1 --2- 172.21.15.78:0/2294944221 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f9c9406ff10 0x7f9c9412ca30 unknown :-1 s=CLOSED pgs=58 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:02.925 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.923+0000 7f9c9ab9c640 1 --2- 172.21.15.78:0/2294944221 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f9c9406ecd0 0x7f9c94134330 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:02.925 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.923+0000 7f9c9ab9c640 1 -- 172.21.15.78:0/2294944221 >> 172.21.15.78:0/2294944221 conn(0x7f9c9409c980 msgr2=0x7f9c9409cd90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:21:02.925 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.923+0000 7f9c9ab9c640 1 -- 172.21.15.78:0/2294944221 shutdown_connections 2024-09-21T09:21:02.925 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.923+0000 7f9c9ab9c640 1 -- 172.21.15.78:0/2294944221 wait complete. 2024-09-21T09:21:02.927 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.923+0000 7f9c9ab9c640 1 Processor -- start 2024-09-21T09:21:02.927 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.924+0000 7f9c9ab9c640 1 -- start start 2024-09-21T09:21:02.927 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.924+0000 7f9c9ab9c640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f9c9406ecd0 0x7f9c94125360 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:21:02.927 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.924+0000 7f9c9ab9c640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f9c9406ff10 0x7f9c941258a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:21:02.927 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.924+0000 7f9c9ab9c640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f9c9412c650 0x7f9c94125de0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:21:02.928 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.924+0000 7f9c9ab9c640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9c94128a40 con 0x7f9c9412c650 2024-09-21T09:21:02.928 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.924+0000 7f9c9ab9c640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9c94127650 con 0x7f9c9406ecd0 2024-09-21T09:21:02.928 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.924+0000 7f9c9ab9c640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 v1 -- 0x7f9c940712d0 con 0x7f9c9406ff10 2024-09-21T09:21:02.928 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.924+0000 7f9c93fff640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f9c9406ff10 0x7f9c941258a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:21:02.928 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.924+0000 7f9c98938640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f9c9406ecd0 0x7f9c94125360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:21:02.928 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.924+0000 7f9c93fff640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f9c9406ff10 0x7f9c941258a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3302/0 says I am v2:172.21.15.78:46316/0 (socket says 172.21.15.78:46316) 2024-09-21T09:21:02.928 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.924+0000 7f9c99139640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f9c9412c650 0x7f9c94125de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:21:02.928 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.924+0000 7f9c98938640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f9c9406ecd0 0x7f9c94125360 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3301/0 says I am v2:172.21.15.78:58270/0 (socket says 172.21.15.78:58270) 2024-09-21T09:21:02.928 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.924+0000 7f9c93fff640 1 -- 172.21.15.78:0/2709090517 learned_addr learned my addr 172.21.15.78:0/2709090517 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:21:02.928 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.924+0000 7f9c93fff640 1 -- 172.21.15.78:0/2709090517 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f9c9406ecd0 msgr2=0x7f9c94125360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:21:02.928 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.924+0000 7f9c93fff640 1 --2- 172.21.15.78:0/2709090517 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f9c9406ecd0 0x7f9c94125360 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:02.928 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.924+0000 7f9c93fff640 1 -- 172.21.15.78:0/2709090517 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f9c9412c650 msgr2=0x7f9c94125de0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:21:02.929 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.924+0000 7f9c93fff640 1 --2- 172.21.15.78:0/2709090517 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f9c9412c650 0x7f9c94125de0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:02.929 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.924+0000 7f9c93fff640 1 -- 172.21.15.78:0/2709090517 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9c8c009840 con 0x7f9c9406ff10 2024-09-21T09:21:02.929 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.925+0000 7f9c93fff640 1 --2- 172.21.15.78:0/2709090517 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f9c9406ff10 0x7f9c941258a0 secure :-1 s=READY pgs=43 cs=0 l=1 rev1=1 crypto rx=0x7f9c80006010 tx=0x7f9c80003820 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:21:02.929 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.925+0000 7f9c917fa640 1 -- 172.21.15.78:0/2709090517 <== mon.2 v2:172.21.15.78:3302/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9c80003e20 con 0x7f9c9406ff10 2024-09-21T09:21:02.929 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.925+0000 7f9c917fa640 1 -- 172.21.15.78:0/2709090517 <== mon.2 v2:172.21.15.78:3302/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7f9c8003c580 con 0x7f9c9406ff10 2024-09-21T09:21:02.929 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.925+0000 7f9c9ab9c640 1 -- 172.21.15.78:0/2709090517 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9c80009560 con 0x7f9c9406ff10 2024-09-21T09:21:02.929 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.925+0000 7f9c917fa640 1 -- 172.21.15.78:0/2709090517 <== mon.2 v2:172.21.15.78:3302/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9c8003c990 con 0x7f9c9406ff10 2024-09-21T09:21:02.929 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.925+0000 7f9c9ab9c640 1 -- 172.21.15.78:0/2709090517 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9c941265a0 con 0x7f9c9406ff10 2024-09-21T09:21:02.929 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.926+0000 7f9c917fa640 1 -- 172.21.15.78:0/2709090517 <== mon.2 v2:172.21.15.78:3302/0 4 ==== mgrmap(e 5) v1 ==== 55212+0+0 (secure 0 0 0) 0x7f9c8003caf0 con 0x7f9c9406ff10 2024-09-21T09:21:02.929 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.926+0000 7f9c9ab9c640 1 -- 172.21.15.78:0/2709090517 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9c9412ca30 con 0x7f9c9406ff10 2024-09-21T09:21:02.929 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.926+0000 7f9c917fa640 1 -- 172.21.15.78:0/2709090517 <== mon.2 v2:172.21.15.78:3302/0 5 ==== osd_map(17..17 src has 1..17) v4 ==== 2868+0+0 (secure 0 0 0) 0x7f9c8007dc40 con 0x7f9c9406ff10 2024-09-21T09:21:02.931 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:02.930+0000 7f9c917fa640 1 -- 172.21.15.78:0/2709090517 <== mon.2 v2:172.21.15.78:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+146310 (secure 0 0 0) 0x7f9c80040080 con 0x7f9c9406ff10 2024-09-21T09:21:03.064 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.062+0000 7f9c9ab9c640 1 -- 172.21.15.78:0/2709090517 --> v1:172.21.15.78:6816/2602406893 -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7f9c94065cc0 con 0x7f9c78047560 2024-09-21T09:21:03.065 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.063+0000 7f9c917fa640 1 -- 172.21.15.78:0/2709090517 <== mgr.4106 v1:172.21.15.78:6816/2602406893 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+34942 (unknown 2965378022 0 2833961459) 0x7f9c94065cc0 con 0x7f9c78047560 2024-09-21T09:21:03.065 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:21:03.065 INFO:teuthology.orchestra.run.smithi078.stderr:dumped all 2024-09-21T09:21:03.067 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.065+0000 7f9c9ab9c640 1 -- 172.21.15.78:0/2709090517 >> v1:172.21.15.78:6816/2602406893 conn(0x7f9c78047560 legacy=0x7f9c78049a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:21:03.067 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.065+0000 7f9c9ab9c640 1 -- 172.21.15.78:0/2709090517 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f9c9406ff10 msgr2=0x7f9c941258a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:21:03.067 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.065+0000 7f9c9ab9c640 1 --2- 172.21.15.78:0/2709090517 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f9c9406ff10 0x7f9c941258a0 secure :-1 s=READY pgs=43 cs=0 l=1 rev1=1 crypto rx=0x7f9c80006010 tx=0x7f9c80003820 comp rx=0 tx=0).stop 2024-09-21T09:21:03.067 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.065+0000 7f9c9ab9c640 1 -- 172.21.15.78:0/2709090517 shutdown_connections 2024-09-21T09:21:03.067 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.065+0000 7f9c9ab9c640 1 --2- 172.21.15.78:0/2709090517 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f9c9412c650 0x7f9c94125de0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:03.067 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.065+0000 7f9c9ab9c640 1 --2- 172.21.15.78:0/2709090517 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f9c9406ff10 0x7f9c941258a0 unknown :-1 s=CLOSED pgs=43 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:03.067 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.065+0000 7f9c9ab9c640 1 --2- 172.21.15.78:0/2709090517 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f9c9406ecd0 0x7f9c94125360 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:03.067 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.065+0000 7f9c9ab9c640 1 -- 172.21.15.78:0/2709090517 >> 172.21.15.78:0/2709090517 conn(0x7f9c9409c980 msgr2=0x7f9c94096750 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:21:03.067 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.065+0000 7f9c9ab9c640 1 -- 172.21.15.78:0/2709090517 shutdown_connections 2024-09-21T09:21:03.067 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.065+0000 7f9c9ab9c640 1 -- 172.21.15.78:0/2709090517 wait complete. 2024-09-21T09:21:03.077 INFO:teuthology.orchestra.run.smithi078.stdout:{"pg_ready":true,"pg_map":{"version":20,"stamp":"2024-09-21T09:21:01.141659+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":139,"num_write_kb":1896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100,"ondisk_log_size":100,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":15,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":377487360,"kb_used":109844,"kb_used_data":1744,"kb_used_omap":6,"kb_used_meta":108025,"kb_avail":377377516,"statfs":{"total":386547056640,"available":386434576384,"internally_reserved":0,"allocated":1785856,"data_stored":1345730,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6374,"internal_metadata":110618394},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"5.828556"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":15,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-09-21T09:20:58.335685+0000","last_change":"2024-09-21T09:20:58.335685+0000","last_active":"2024-09-21T09:20:58.335685+0000","last_peered":"2024-09-21T09:20:58.335685+0000","last_clean":"2024-09-21T09:20:58.335685+0000","last_became_active":"2024-09-21T09:20:56.322152+0000","last_became_peered":"2024-09-21T09:20:56.322152+0000","last_unstale":"2024-09-21T09:20:58.335685+0000","last_undegraded":"2024-09-21T09:20:58.335685+0000","last_fullsized":"2024-09-21T09:20:58.335685+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:19:22.863125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039605,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":15,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-09-21T09:20:58.335404+0000","last_change":"2024-09-21T09:20:58.335404+0000","last_active":"2024-09-21T09:20:58.335404+0000","last_peered":"2024-09-21T09:20:58.335404+0000","last_clean":"2024-09-21T09:20:58.335404+0000","last_became_active":"2024-09-21T09:20:56.320383+0000","last_became_peered":"2024-09-21T09:20:56.320383+0000","last_unstale":"2024-09-21T09:20:58.335404+0000","last_undegraded":"2024-09-21T09:20:58.335404+0000","last_fullsized":"2024-09-21T09:20:58.335404+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:36:03.357395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040851400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":15,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-09-21T09:20:58.335829+0000","last_change":"2024-09-21T09:20:58.335829+0000","last_active":"2024-09-21T09:20:58.335829+0000","last_peered":"2024-09-21T09:20:58.335829+0000","last_clean":"2024-09-21T09:20:58.335829+0000","last_became_active":"2024-09-21T09:20:56.322789+0000","last_became_peered":"2024-09-21T09:20:56.322789+0000","last_unstale":"2024-09-21T09:20:58.335829+0000","last_undegraded":"2024-09-21T09:20:58.335829+0000","last_fullsized":"2024-09-21T09:20:58.335829+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:16:32.684631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001005517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":15,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-09-21T09:20:58.335406+0000","last_change":"2024-09-21T09:20:58.335406+0000","last_active":"2024-09-21T09:20:58.335406+0000","last_peered":"2024-09-21T09:20:58.335406+0000","last_clean":"2024-09-21T09:20:58.335406+0000","last_became_active":"2024-09-21T09:20:56.324636+0000","last_became_peered":"2024-09-21T09:20:56.324636+0000","last_unstale":"2024-09-21T09:20:58.335406+0000","last_undegraded":"2024-09-21T09:20:58.335406+0000","last_fullsized":"2024-09-21T09:20:58.335406+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:19:22.863125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054560899999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"17'2","reported_seq":17,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-09-21T09:20:58.343773+0000","last_change":"2024-09-21T09:20:58.336768+0000","last_active":"2024-09-21T09:20:58.343773+0000","last_peered":"2024-09-21T09:20:58.343773+0000","last_clean":"2024-09-21T09:20:58.343773+0000","last_became_active":"2024-09-21T09:20:56.327106+0000","last_became_peered":"2024-09-21T09:20:56.327106+0000","last_unstale":"2024-09-21T09:20:58.343773+0000","last_undegraded":"2024-09-21T09:20:58.343773+0000","last_fullsized":"2024-09-21T09:20:58.343773+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:45:36.558638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011881540000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":15,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-09-21T09:20:58.479452+0000","last_change":"2024-09-21T09:20:58.479452+0000","last_active":"2024-09-21T09:20:58.479452+0000","last_peered":"2024-09-21T09:20:58.479452+0000","last_clean":"2024-09-21T09:20:58.479452+0000","last_became_active":"2024-09-21T09:20:56.324347+0000","last_became_peered":"2024-09-21T09:20:56.324347+0000","last_unstale":"2024-09-21T09:20:58.479452+0000","last_undegraded":"2024-09-21T09:20:58.479452+0000","last_fullsized":"2024-09-21T09:20:58.479452+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:43:56.319711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023051899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":15,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-09-21T09:20:58.335174+0000","last_change":"2024-09-21T09:20:58.335174+0000","last_active":"2024-09-21T09:20:58.335174+0000","last_peered":"2024-09-21T09:20:58.335174+0000","last_clean":"2024-09-21T09:20:58.335174+0000","last_became_active":"2024-09-21T09:20:56.322710+0000","last_became_peered":"2024-09-21T09:20:56.322710+0000","last_unstale":"2024-09-21T09:20:58.335174+0000","last_undegraded":"2024-09-21T09:20:58.335174+0000","last_fullsized":"2024-09-21T09:20:58.335174+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:03:03.617340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039429200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"15'1","reported_seq":16,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-09-21T09:20:58.336756+0000","last_change":"2024-09-21T09:20:58.336756+0000","last_active":"2024-09-21T09:20:58.336756+0000","last_peered":"2024-09-21T09:20:58.336756+0000","last_clean":"2024-09-21T09:20:58.336756+0000","last_became_active":"2024-09-21T09:20:56.321532+0000","last_became_peered":"2024-09-21T09:20:56.321532+0000","last_unstale":"2024-09-21T09:20:58.336756+0000","last_undegraded":"2024-09-21T09:20:58.336756+0000","last_fullsized":"2024-09-21T09:20:58.336756+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:57:55.907381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091827900000000004,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"12'97","reported_seq":141,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-21T09:20:57.328245+0000","last_change":"2024-09-21T09:20:52.294610+0000","last_active":"2024-09-21T09:20:57.328245+0000","last_peered":"2024-09-21T09:20:57.328245+0000","last_clean":"2024-09-21T09:20:57.328245+0000","last_became_active":"2024-09-21T09:20:52.293626+0000","last_became_peered":"2024-09-21T09:20:52.293626+0000","last_unstale":"2024-09-21T09:20:57.328245+0000","last_undegraded":"2024-09-21T09:20:57.328245+0000","last_fullsized":"2024-09-21T09:20:57.328245+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:51.278939+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:51.278939+0000","last_clean_scrub_stamp":"2024-09-21T09:20:51.278939+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:43:02.095040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":4},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97,"ondisk_log_size":97,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":3,"up_from":9,"seq":38654705669,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27172,"kb_used_data":148,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344668,"statfs":{"total":96636764160,"available":96608940032,"internally_reserved":0,"allocated":151552,"data_stored":41258,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":27654600},"hb_peers":[0,1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.38400000000000001}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45000000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46800000000000003}]}]},{"osd":2,"up_from":9,"seq":38654705671,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27168,"kb_used_data":144,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344672,"statfs":{"total":96636764160,"available":96608944128,"internally_reserved":0,"allocated":147456,"data_stored":41239,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.33600000000000002}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26200000000000001}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.248}]}]},{"osd":1,"up_from":9,"seq":38654705669,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27756,"kb_used_data":728,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344084,"statfs":{"total":96636764160,"available":96608342016,"internally_reserved":0,"allocated":745472,"data_stored":631626,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45200000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51500000000000001}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47299999999999998}]}]},{"osd":0,"up_from":9,"seq":38654705669,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27748,"kb_used_data":724,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344092,"statfs":{"total":96636764160,"available":96608350208,"internally_reserved":0,"allocated":741376,"data_stored":631607,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[1,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.255}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.37}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.33000000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T09:21:03.078 INFO:tasks.ceph.ceph_manager.ceph:clean! 2024-09-21T09:21:03.078 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-21T09:21:03.078 INFO:tasks.ceph.ceph_manager.ceph:wait_until_healthy 2024-09-21T09:21:03.078 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph health --format=json 2024-09-21T09:21:03.177 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.176+0000 7f2ad9525640 1 -- 172.21.15.78:0/3692797274 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f2ad406eee0 msgr2=0x7f2ad406f2e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:21:03.177 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.176+0000 7f2ad9525640 1 --2- 172.21.15.78:0/3692797274 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f2ad406eee0 0x7f2ad406f2e0 secure :-1 s=READY pgs=82 cs=0 l=1 rev1=1 crypto rx=0x7f2abc009830 tx=0x7f2abc02a360 comp rx=0 tx=0).stop 2024-09-21T09:21:03.177 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.176+0000 7f2ad9525640 1 -- 172.21.15.78:0/3692797274 shutdown_connections 2024-09-21T09:21:03.178 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.176+0000 7f2ad9525640 1 --2- 172.21.15.78:0/3692797274 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f2ad406f820 0x7f2ad4131170 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:03.178 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.176+0000 7f2ad9525640 1 --2- 172.21.15.78:0/3692797274 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f2ad406eee0 0x7f2ad406f2e0 unknown :-1 s=CLOSED pgs=82 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:03.178 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.176+0000 7f2ad9525640 1 --2- 172.21.15.78:0/3692797274 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f2ad412a4e0 0x7f2ad412a8c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:03.178 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.176+0000 7f2ad9525640 1 -- 172.21.15.78:0/3692797274 >> 172.21.15.78:0/3692797274 conn(0x7f2ad40972a0 msgr2=0x7f2ad40976b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:21:03.178 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.176+0000 7f2ad9525640 1 -- 172.21.15.78:0/3692797274 shutdown_connections 2024-09-21T09:21:03.178 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.176+0000 7f2ad9525640 1 -- 172.21.15.78:0/3692797274 wait complete. 2024-09-21T09:21:03.180 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.177+0000 7f2ad9525640 1 Processor -- start 2024-09-21T09:21:03.180 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.177+0000 7f2ad9525640 1 -- start start 2024-09-21T09:21:03.180 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.177+0000 7f2ad9525640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f2ad406eee0 0x7f2ad4124200 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:21:03.180 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.177+0000 7f2ad9525640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f2ad406f820 0x7f2ad4124740 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:21:03.181 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.177+0000 7f2ad9525640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f2ad412a4e0 0x7f2ad4124c80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:21:03.181 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.177+0000 7f2ad9525640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2ad4126920 con 0x7f2ad412a4e0 2024-09-21T09:21:03.181 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.177+0000 7f2ad9525640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2ad4096e00 con 0x7f2ad406eee0 2024-09-21T09:21:03.181 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.177+0000 7f2ad9525640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 v1 -- 0x7f2ad4071c10 con 0x7f2ad406f820 2024-09-21T09:21:03.181 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.177+0000 7f2ad2ffd640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f2ad406eee0 0x7f2ad4124200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:21:03.181 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.177+0000 7f2ad2ffd640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f2ad406eee0 0x7f2ad4124200 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3301/0 says I am v2:172.21.15.78:58282/0 (socket says 172.21.15.78:58282) 2024-09-21T09:21:03.181 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.177+0000 7f2ad2ffd640 1 -- 172.21.15.78:0/2049658320 learned_addr learned my addr 172.21.15.78:0/2049658320 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:21:03.181 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.177+0000 7f2ad27fc640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f2ad406f820 0x7f2ad4124740 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:21:03.181 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.177+0000 7f2ad37fe640 1 --2- 172.21.15.78:0/2049658320 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f2ad412a4e0 0x7f2ad4124c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:21:03.181 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.177+0000 7f2ad2ffd640 1 -- 172.21.15.78:0/2049658320 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f2ad406f820 msgr2=0x7f2ad4124740 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:21:03.181 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.177+0000 7f2ad2ffd640 1 --2- 172.21.15.78:0/2049658320 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f2ad406f820 0x7f2ad4124740 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:03.181 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.177+0000 7f2ad2ffd640 1 -- 172.21.15.78:0/2049658320 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f2ad412a4e0 msgr2=0x7f2ad4124c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:21:03.181 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.177+0000 7f2ad2ffd640 1 --2- 172.21.15.78:0/2049658320 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f2ad412a4e0 0x7f2ad4124c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:03.181 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.177+0000 7f2ad2ffd640 1 -- 172.21.15.78:0/2049658320 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2abc009490 con 0x7f2ad406eee0 2024-09-21T09:21:03.182 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.177+0000 7f2ad37fe640 1 --2- 172.21.15.78:0/2049658320 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f2ad412a4e0 0x7f2ad4124c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:21:03.182 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.177+0000 7f2ad27fc640 1 --2- 172.21.15.78:0/2049658320 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f2ad406f820 0x7f2ad4124740 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:21:03.182 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.178+0000 7f2ad2ffd640 1 --2- 172.21.15.78:0/2049658320 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f2ad406eee0 0x7f2ad4124200 secure :-1 s=READY pgs=59 cs=0 l=1 rev1=1 crypto rx=0x7f2acc00e9c0 tx=0x7f2acc00edc0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:21:03.182 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.178+0000 7f2acb7fe640 1 -- 172.21.15.78:0/2049658320 <== mon.1 v2:172.21.15.78:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2acc00ccc0 con 0x7f2ad406eee0 2024-09-21T09:21:03.182 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.178+0000 7f2acb7fe640 1 -- 172.21.15.78:0/2049658320 <== mon.1 v2:172.21.15.78:3301/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7f2acc00ce20 con 0x7f2ad406eee0 2024-09-21T09:21:03.182 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.178+0000 7f2ad9525640 1 -- 172.21.15.78:0/2049658320 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f2ad4127070 con 0x7f2ad406eee0 2024-09-21T09:21:03.182 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.178+0000 7f2acb7fe640 1 -- 172.21.15.78:0/2049658320 <== mon.1 v2:172.21.15.78:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2acc005280 con 0x7f2ad406eee0 2024-09-21T09:21:03.182 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.178+0000 7f2ad9525640 1 -- 172.21.15.78:0/2049658320 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2ad41253a0 con 0x7f2ad406eee0 2024-09-21T09:21:03.182 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.179+0000 7f2acb7fe640 1 -- 172.21.15.78:0/2049658320 <== mon.1 v2:172.21.15.78:3301/0 4 ==== mgrmap(e 5) v1 ==== 55212+0+0 (secure 0 0 0) 0x7f2acc0053e0 con 0x7f2ad406eee0 2024-09-21T09:21:03.182 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.179+0000 7f2ad9525640 1 -- 172.21.15.78:0/2049658320 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f2ad4126cd0 con 0x7f2ad406eee0 2024-09-21T09:21:03.182 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.179+0000 7f2acb7fe640 1 -- 172.21.15.78:0/2049658320 <== mon.1 v2:172.21.15.78:3301/0 5 ==== osd_map(18..18 src has 1..18) v4 ==== 2840+0+0 (secure 0 0 0) 0x7f2acc014070 con 0x7f2ad406eee0 2024-09-21T09:21:03.184 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.183+0000 7f2acb7fe640 1 -- 172.21.15.78:0/2049658320 <== mon.1 v2:172.21.15.78:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+146310 (secure 0 0 0) 0x7f2ad4126cd0 con 0x7f2ad406eee0 2024-09-21T09:21:03.344 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.342+0000 7f2ad9525640 1 -- 172.21.15.78:0/2049658320 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_command({"prefix": "health", "format": "json"} v 0) v1 -- 0x7f2ad4126cd0 con 0x7f2ad406eee0 2024-09-21T09:21:03.344 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.342+0000 7f2acb7fe640 1 -- 172.21.15.78:0/2049658320 <== mon.1 v2:172.21.15.78:3301/0 7 ==== mon_command_ack([{"prefix": "health", "format": "json"}]=0 v0) v1 ==== 72+0+46 (secure 0 0 0) 0x7f2ad4126cd0 con 0x7f2ad406eee0 2024-09-21T09:21:03.344 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:21:03.344 INFO:teuthology.orchestra.run.smithi078.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-21T09:21:03.346 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.344+0000 7f2ad9525640 1 -- 172.21.15.78:0/2049658320 >> v1:172.21.15.78:6816/2602406893 conn(0x7f2ab4047560 legacy=0x7f2ab4049a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:21:03.346 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.344+0000 7f2ad9525640 1 -- 172.21.15.78:0/2049658320 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f2ad406eee0 msgr2=0x7f2ad4124200 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:21:03.346 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.344+0000 7f2ad9525640 1 --2- 172.21.15.78:0/2049658320 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f2ad406eee0 0x7f2ad4124200 secure :-1 s=READY pgs=59 cs=0 l=1 rev1=1 crypto rx=0x7f2acc00e9c0 tx=0x7f2acc00edc0 comp rx=0 tx=0).stop 2024-09-21T09:21:03.346 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.344+0000 7f2ad9525640 1 -- 172.21.15.78:0/2049658320 shutdown_connections 2024-09-21T09:21:03.346 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.344+0000 7f2ad9525640 1 --2- 172.21.15.78:0/2049658320 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f2ad412a4e0 0x7f2ad4124c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:03.346 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.344+0000 7f2ad9525640 1 --2- 172.21.15.78:0/2049658320 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f2ad406f820 0x7f2ad4124740 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:03.346 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.344+0000 7f2ad9525640 1 --2- 172.21.15.78:0/2049658320 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f2ad406eee0 0x7f2ad4124200 unknown :-1 s=CLOSED pgs=59 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:03.346 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.344+0000 7f2ad9525640 1 -- 172.21.15.78:0/2049658320 >> 172.21.15.78:0/2049658320 conn(0x7f2ad40972a0 msgr2=0x7f2ad41306c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:21:03.347 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.344+0000 7f2ad9525640 1 -- 172.21.15.78:0/2049658320 shutdown_connections 2024-09-21T09:21:03.347 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.344+0000 7f2ad9525640 1 -- 172.21.15.78:0/2049658320 wait complete. 2024-09-21T09:21:03.356 INFO:tasks.ceph.ceph_manager.ceph:wait_until_healthy done 2024-09-21T09:21:03.356 INFO:teuthology.run_tasks:Running task exec... 2024-09-21T09:21:03.365 INFO:teuthology.task.exec:Executing custom commands... 2024-09-21T09:21:03.365 INFO:teuthology.task.exec:Running commands on role osd.0 host ubuntu@smithi078.front.sepia.ceph.com 2024-09-21T09:21:03.366 DEBUG:teuthology.orchestra.run.smithi078:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph osd set-require-min-compat-client reef' 2024-09-21T09:21:03.463 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.462+0000 7f49932e5640 1 -- 172.21.15.78:0/1671691930 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f498c06f010 msgr2=0x7f498c12e020 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:21:03.463 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.462+0000 7f49932e5640 1 --2- 172.21.15.78:0/1671691930 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f498c06f010 0x7f498c12e020 secure :-1 s=READY pgs=84 cs=0 l=1 rev1=1 crypto rx=0x7f4980005fd0 tx=0x7f498002a820 comp rx=0 tx=0).stop 2024-09-21T09:21:03.463 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.462+0000 7f49932e5640 1 -- 172.21.15.78:0/1671691930 shutdown_connections 2024-09-21T09:21:03.463 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.462+0000 7f49932e5640 1 --2- 172.21.15.78:0/1671691930 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f498c06f010 0x7f498c12e020 unknown :-1 s=CLOSED pgs=84 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:03.463 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.462+0000 7f49932e5640 1 --2- 172.21.15.78:0/1671691930 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f498c12e560 0x7f498c130950 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:03.463 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.462+0000 7f49932e5640 1 -- 172.21.15.78:0/1671691930 >> 172.21.15.78:0/1671691930 conn(0x7f498c097510 msgr2=0x7f498c097920 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:21:03.464 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.462+0000 7f49932e5640 1 -- 172.21.15.78:0/1671691930 shutdown_connections 2024-09-21T09:21:03.464 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.462+0000 7f49932e5640 1 -- 172.21.15.78:0/1671691930 wait complete. 2024-09-21T09:21:03.465 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.462+0000 7f49932e5640 1 Processor -- start 2024-09-21T09:21:03.465 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.463+0000 7f49932e5640 1 -- start start 2024-09-21T09:21:03.465 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.463+0000 7f49932e5640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f498c06f010 0x7f498c12cfb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:21:03.466 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.463+0000 7f49932e5640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f498c094440 0x7f498c12d4f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:21:03.466 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.463+0000 7f49932e5640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f498c12e560 0x7f498c12da30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:21:03.466 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.463+0000 7f49932e5640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f498c12df70 con 0x7f498c12e560 2024-09-21T09:21:03.466 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.463+0000 7f49932e5640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f498c12e0e0 con 0x7f498c06f010 2024-09-21T09:21:03.466 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.463+0000 7f49932e5640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 v1 -- 0x7f498c120430 con 0x7f498c094440 2024-09-21T09:21:03.466 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.463+0000 7f4991882640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f498c12e560 0x7f498c12da30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:21:03.466 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.463+0000 7f4990880640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f498c094440 0x7f498c12d4f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:21:03.466 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.463+0000 7f4991081640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f498c06f010 0x7f498c12cfb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:21:03.466 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.463+0000 7f4991882640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f498c12e560 0x7f498c12da30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3300/0 says I am v2:172.21.15.78:37110/0 (socket says 172.21.15.78:37110) 2024-09-21T09:21:03.466 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.463+0000 7f4991882640 1 -- 172.21.15.78:0/2339145879 learned_addr learned my addr 172.21.15.78:0/2339145879 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:21:03.467 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.463+0000 7f4990880640 1 -- 172.21.15.78:0/2339145879 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f498c06f010 msgr2=0x7f498c12cfb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:21:03.467 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.463+0000 7f4990880640 1 --2- 172.21.15.78:0/2339145879 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f498c06f010 0x7f498c12cfb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:03.467 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.463+0000 7f4990880640 1 -- 172.21.15.78:0/2339145879 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f498c12e560 msgr2=0x7f498c12da30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:21:03.467 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.463+0000 7f4990880640 1 --2- 172.21.15.78:0/2339145879 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f498c12e560 0x7f498c12da30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:03.467 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.463+0000 7f4990880640 1 -- 172.21.15.78:0/2339145879 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4984009490 con 0x7f498c094440 2024-09-21T09:21:03.467 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.463+0000 7f4991081640 1 --2- 172.21.15.78:0/2339145879 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f498c06f010 0x7f498c12cfb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:21:03.467 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.463+0000 7f4990880640 1 --2- 172.21.15.78:0/2339145879 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f498c094440 0x7f498c12d4f0 secure :-1 s=READY pgs=44 cs=0 l=1 rev1=1 crypto rx=0x7f498c0982e0 tx=0x7f4980033e40 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:21:03.467 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.464+0000 7f4989ffb640 1 -- 172.21.15.78:0/2339145879 <== mon.2 v2:172.21.15.78:3302/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4980003ea0 con 0x7f498c094440 2024-09-21T09:21:03.467 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.464+0000 7f4989ffb640 1 -- 172.21.15.78:0/2339145879 <== mon.2 v2:172.21.15.78:3302/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7f4980004250 con 0x7f498c094440 2024-09-21T09:21:03.467 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.464+0000 7f49932e5640 1 -- 172.21.15.78:0/2339145879 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4980009840 con 0x7f498c094440 2024-09-21T09:21:03.467 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.464+0000 7f4989ffb640 1 -- 172.21.15.78:0/2339145879 <== mon.2 v2:172.21.15.78:3302/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4980004660 con 0x7f498c094440 2024-09-21T09:21:03.467 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.464+0000 7f49932e5640 1 -- 172.21.15.78:0/2339145879 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f498c1210f0 con 0x7f498c094440 2024-09-21T09:21:03.468 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.465+0000 7f4989ffb640 1 -- 172.21.15.78:0/2339145879 <== mon.2 v2:172.21.15.78:3302/0 4 ==== mgrmap(e 5) v1 ==== 55212+0+0 (secure 0 0 0) 0x7f498003e450 con 0x7f498c094440 2024-09-21T09:21:03.468 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.465+0000 7f49932e5640 1 -- 172.21.15.78:0/2339145879 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f498c094820 con 0x7f498c094440 2024-09-21T09:21:03.468 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.465+0000 7f4989ffb640 1 -- 172.21.15.78:0/2339145879 <== mon.2 v2:172.21.15.78:3302/0 5 ==== osd_map(18..18 src has 1..18) v4 ==== 2840+0+0 (secure 0 0 0) 0x7f498007da60 con 0x7f498c094440 2024-09-21T09:21:03.470 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.469+0000 7f4989ffb640 1 -- 172.21.15.78:0/2339145879 <== mon.2 v2:172.21.15.78:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+146310 (secure 0 0 0) 0x7f4980048cc0 con 0x7f498c094440 2024-09-21T09:21:03.615 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:03.614+0000 7f49932e5640 1 -- 172.21.15.78:0/2339145879 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_command({"prefix": "osd set-require-min-compat-client", "version": "reef"} v 0) v1 -- 0x7f498c1205c0 con 0x7f498c094440 2024-09-21T09:21:04.505 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:04.504+0000 7f4989ffb640 1 -- 172.21.15.78:0/2339145879 <== mon.2 v2:172.21.15.78:3302/0 7 ==== mon_command_ack([{"prefix": "osd set-require-min-compat-client", "version": "reef"}]=0 set require_min_compat_client to reef v19) v1 ==== 137+0+0 (secure 0 0 0) 0x7f4980037070 con 0x7f498c094440 2024-09-21T09:21:04.505 INFO:teuthology.orchestra.run.smithi078.stderr:set require_min_compat_client to reef 2024-09-21T09:21:04.508 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:04.506+0000 7f49932e5640 1 -- 172.21.15.78:0/2339145879 >> v1:172.21.15.78:6816/2602406893 conn(0x7f4978047560 legacy=0x7f4978049a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:21:04.508 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:04.506+0000 7f49932e5640 1 -- 172.21.15.78:0/2339145879 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f498c094440 msgr2=0x7f498c12d4f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:21:04.508 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:04.506+0000 7f49932e5640 1 --2- 172.21.15.78:0/2339145879 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f498c094440 0x7f498c12d4f0 secure :-1 s=READY pgs=44 cs=0 l=1 rev1=1 crypto rx=0x7f498c0982e0 tx=0x7f4980033e40 comp rx=0 tx=0).stop 2024-09-21T09:21:04.508 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:04.506+0000 7f49932e5640 1 -- 172.21.15.78:0/2339145879 shutdown_connections 2024-09-21T09:21:04.508 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:04.506+0000 7f49932e5640 1 --2- 172.21.15.78:0/2339145879 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f498c12e560 0x7f498c12da30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:04.508 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:04.506+0000 7f49932e5640 1 --2- 172.21.15.78:0/2339145879 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f498c094440 0x7f498c12d4f0 unknown :-1 s=CLOSED pgs=44 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:04.508 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:04.506+0000 7f49932e5640 1 --2- 172.21.15.78:0/2339145879 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f498c06f010 0x7f498c12cfb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:04.508 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:04.506+0000 7f49932e5640 1 -- 172.21.15.78:0/2339145879 >> 172.21.15.78:0/2339145879 conn(0x7f498c097510 msgr2=0x7f498c06c7e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:21:04.509 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:04.506+0000 7f49932e5640 1 -- 172.21.15.78:0/2339145879 shutdown_connections 2024-09-21T09:21:04.509 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:04.506+0000 7f49932e5640 1 -- 172.21.15.78:0/2339145879 wait complete. 2024-09-21T09:21:04.520 INFO:teuthology.run_tasks:Running task print... 2024-09-21T09:21:04.529 INFO:teuthology.task.print:**** done ceph 2024-09-21T09:21:04.530 INFO:teuthology.run_tasks:Running task print... 2024-09-21T09:21:04.550 INFO:teuthology.task.print:*** upgrading, no cephfs present 2024-09-21T09:21:04.550 INFO:teuthology.run_tasks:Running task exec... 2024-09-21T09:21:04.558 INFO:teuthology.task.exec:Executing custom commands... 2024-09-21T09:21:04.558 INFO:teuthology.task.exec:Running commands on role mon.a host ubuntu@smithi078.front.sepia.ceph.com 2024-09-21T09:21:04.558 DEBUG:teuthology.orchestra.run.smithi078:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph fs dump' 2024-09-21T09:21:04.664 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:04.662+0000 7f00d019c640 1 -- 172.21.15.78:0/3924176657 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f00c806d420 msgr2=0x7f00c81312a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:21:04.664 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:04.662+0000 7f00d019c640 1 --2- 172.21.15.78:0/3924176657 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f00c806d420 0x7f00c81312a0 secure :-1 s=READY pgs=45 cs=0 l=1 rev1=1 crypto rx=0x7f00bc009830 tx=0x7f00bc02a480 comp rx=0 tx=0).stop 2024-09-21T09:21:04.664 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:04.662+0000 7f00d019c640 1 -- 172.21.15.78:0/3924176657 shutdown_connections 2024-09-21T09:21:04.664 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:04.662+0000 7f00d019c640 1 --2- 172.21.15.78:0/3924176657 >> v2:172.21.15.78:3300/0 conn(0x7f00c81317e0 0x7f00c8133bd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:04.664 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:04.662+0000 7f00d019c640 1 --2- 172.21.15.78:0/3924176657 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f00c806d420 0x7f00c81312a0 unknown :-1 s=CLOSED pgs=45 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:04.664 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:04.662+0000 7f00d019c640 1 -- 172.21.15.78:0/3924176657 >> 172.21.15.78:0/3924176657 conn(0x7f00c809caa0 msgr2=0x7f00c809ceb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:21:04.664 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:04.662+0000 7f00d019c640 1 -- 172.21.15.78:0/3924176657 shutdown_connections 2024-09-21T09:21:04.664 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:04.662+0000 7f00d019c640 1 -- 172.21.15.78:0/3924176657 wait complete. 2024-09-21T09:21:04.665 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:04.663+0000 7f00d019c640 1 Processor -- start 2024-09-21T09:21:04.665 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:04.663+0000 7f00d019c640 1 -- start start 2024-09-21T09:21:04.667 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:04.663+0000 7f00d019c640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f00c806c930 0x7f00c8123c60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:21:04.667 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:04.663+0000 7f00d019c640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f00c806d420 0x7f00c81241a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:21:04.667 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:04.663+0000 7f00d019c640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f00c81317e0 0x7f00c81246e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:21:04.667 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:04.663+0000 7f00d019c640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f00c8125080 con 0x7f00c806c930 2024-09-21T09:21:04.667 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:04.663+0000 7f00d019c640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f00c8125830 con 0x7f00c81317e0 2024-09-21T09:21:04.667 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:04.663+0000 7f00d019c640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 v1 -- 0x7f00c8130cf0 con 0x7f00c806d420 2024-09-21T09:21:04.667 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:04.663+0000 7f00cd737640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f00c806d420 0x7f00c81241a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:21:04.667 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:04.663+0000 7f00cdf38640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f00c806c930 0x7f00c8123c60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:21:04.667 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:04.663+0000 7f00cd737640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f00c806d420 0x7f00c81241a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3302/0 says I am v2:172.21.15.78:46380/0 (socket says 172.21.15.78:46380) 2024-09-21T09:21:04.668 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:04.663+0000 7f00ce739640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f00c81317e0 0x7f00c81246e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:21:04.668 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:04.663+0000 7f00cdf38640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f00c806c930 0x7f00c8123c60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3300/0 says I am v2:172.21.15.78:37116/0 (socket says 172.21.15.78:37116) 2024-09-21T09:21:04.668 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:04.663+0000 7f00cd737640 1 -- 172.21.15.78:0/4189858387 learned_addr learned my addr 172.21.15.78:0/4189858387 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:21:04.668 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:04.664+0000 7f00cdf38640 1 -- 172.21.15.78:0/4189858387 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f00c806d420 msgr2=0x7f00c81241a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:21:04.668 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:04.664+0000 7f00cdf38640 1 --2- 172.21.15.78:0/4189858387 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f00c806d420 0x7f00c81241a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:04.668 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:04.664+0000 7f00cdf38640 1 -- 172.21.15.78:0/4189858387 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f00c81317e0 msgr2=0x7f00c81246e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:21:04.668 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:04.664+0000 7f00cdf38640 1 --2- 172.21.15.78:0/4189858387 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f00c81317e0 0x7f00c81246e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:04.668 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:04.664+0000 7f00cdf38640 1 -- 172.21.15.78:0/4189858387 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f00b8009840 con 0x7f00c806c930 2024-09-21T09:21:04.669 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:04.664+0000 7f00cd737640 1 --2- 172.21.15.78:0/4189858387 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f00c806d420 0x7f00c81241a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:21:04.669 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:04.664+0000 7f00cdf38640 1 --2- 172.21.15.78:0/4189858387 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f00c806c930 0x7f00c8123c60 secure :-1 s=READY pgs=86 cs=0 l=1 rev1=1 crypto rx=0x7f00c000c3d0 tx=0x7f00c000c890 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:21:04.669 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:04.664+0000 7f00c67fc640 1 -- 172.21.15.78:0/4189858387 <== mon.0 v2:172.21.15.78:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f00c0011790 con 0x7f00c806c930 2024-09-21T09:21:04.669 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:04.664+0000 7f00c67fc640 1 -- 172.21.15.78:0/4189858387 <== mon.0 v2:172.21.15.78:3300/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7f00c00119e0 con 0x7f00c806c930 2024-09-21T09:21:04.669 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:04.664+0000 7f00d019c640 1 -- 172.21.15.78:0/4189858387 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f00bc009490 con 0x7f00c806c930 2024-09-21T09:21:04.669 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:04.664+0000 7f00c67fc640 1 -- 172.21.15.78:0/4189858387 <== mon.0 v2:172.21.15.78:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f00c0011df0 con 0x7f00c806c930 2024-09-21T09:21:04.669 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:04.664+0000 7f00d019c640 1 -- 172.21.15.78:0/4189858387 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f00c81262a0 con 0x7f00c806c930 2024-09-21T09:21:04.669 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:04.665+0000 7f00c67fc640 1 -- 172.21.15.78:0/4189858387 <== mon.0 v2:172.21.15.78:3300/0 4 ==== mgrmap(e 5) v1 ==== 55212+0+0 (secure 0 0 0) 0x7f00c001c460 con 0x7f00c806c930 2024-09-21T09:21:04.669 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:04.665+0000 7f00d019c640 1 -- 172.21.15.78:0/4189858387 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f00c806cd10 con 0x7f00c806c930 2024-09-21T09:21:04.669 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:04.665+0000 7f00c67fc640 1 -- 172.21.15.78:0/4189858387 <== mon.0 v2:172.21.15.78:3300/0 5 ==== osd_map(19..19 src has 1..19) v4 ==== 2812+0+0 (secure 0 0 0) 0x7f00c005c6e0 con 0x7f00c806c930 2024-09-21T09:21:04.670 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:04.669+0000 7f00c67fc640 1 -- 172.21.15.78:0/4189858387 <== mon.0 v2:172.21.15.78:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+146310 (secure 0 0 0) 0x7f00c003a3d0 con 0x7f00c806c930 2024-09-21T09:21:04.826 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:04.825+0000 7f00d019c640 1 -- 172.21.15.78:0/4189858387 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "fs dump"} v 0) v1 -- 0x7f00c8066120 con 0x7f00c806c930 2024-09-21T09:21:04.827 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:04.825+0000 7f00c67fc640 1 -- 172.21.15.78:0/4189858387 <== mon.0 v2:172.21.15.78:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump"}]=0 dumped fsmap epoch 1 v1) v1 ==== 75+0+358 (secure 0 0 0) 0x7f00c003d9f0 con 0x7f00c806c930 2024-09-21T09:21:04.827 INFO:teuthology.orchestra.run.smithi078.stdout:e1 2024-09-21T09:21:04.827 INFO:teuthology.orchestra.run.smithi078.stdout:enable_multiple, ever_enabled_multiple: 1,1 2024-09-21T09:21:04.827 INFO:teuthology.orchestra.run.smithi078.stdout:default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-21T09:21:04.827 INFO:teuthology.orchestra.run.smithi078.stdout:legacy client fscid: -1 2024-09-21T09:21:04.827 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:21:04.827 INFO:teuthology.orchestra.run.smithi078.stdout:No filesystems configured 2024-09-21T09:21:04.827 INFO:teuthology.orchestra.run.smithi078.stderr:dumped fsmap epoch 1 2024-09-21T09:21:04.828 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:04.827+0000 7f00d019c640 1 -- 172.21.15.78:0/4189858387 >> v1:172.21.15.78:6816/2602406893 conn(0x7f00b4047560 legacy=0x7f00b4049a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:21:04.828 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:04.827+0000 7f00d019c640 1 -- 172.21.15.78:0/4189858387 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f00c806c930 msgr2=0x7f00c8123c60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:21:04.829 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:04.827+0000 7f00d019c640 1 --2- 172.21.15.78:0/4189858387 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f00c806c930 0x7f00c8123c60 secure :-1 s=READY pgs=86 cs=0 l=1 rev1=1 crypto rx=0x7f00c000c3d0 tx=0x7f00c000c890 comp rx=0 tx=0).stop 2024-09-21T09:21:04.829 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:04.827+0000 7f00d019c640 1 -- 172.21.15.78:0/4189858387 shutdown_connections 2024-09-21T09:21:04.829 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:04.827+0000 7f00d019c640 1 --2- 172.21.15.78:0/4189858387 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f00c81317e0 0x7f00c81246e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:04.829 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:04.827+0000 7f00d019c640 1 --2- 172.21.15.78:0/4189858387 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f00c806d420 0x7f00c81241a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:04.829 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:04.827+0000 7f00d019c640 1 --2- 172.21.15.78:0/4189858387 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f00c806c930 0x7f00c8123c60 unknown :-1 s=CLOSED pgs=86 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:21:04.829 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:04.827+0000 7f00d019c640 1 -- 172.21.15.78:0/4189858387 >> 172.21.15.78:0/4189858387 conn(0x7f00c809caa0 msgr2=0x7f00c812bee0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:21:04.829 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:04.827+0000 7f00d019c640 1 -- 172.21.15.78:0/4189858387 shutdown_connections 2024-09-21T09:21:04.829 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:21:04.827+0000 7f00d019c640 1 -- 172.21.15.78:0/4189858387 wait complete. 2024-09-21T09:21:04.838 INFO:teuthology.run_tasks:Running task install.upgrade... 2024-09-21T09:21:04.853 INFO:teuthology.task.install:project ceph config {'mon.a': None} overrides {'flavor': 'default', 'sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208'} 2024-09-21T09:21:04.853 INFO:teuthology.task.install:remote ubuntu@smithi078.front.sepia.ceph.com config {'flavor': 'default', 'sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208'} 2024-09-21T09:21:04.853 INFO:teuthology.task.install:extra packages: [] 2024-09-21T09:21:04.859 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'ceph-volume', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'ceph-volume', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2024-09-21T09:21:04.859 INFO:teuthology.task.install:Upgrading ceph rpm packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, ceph-volume, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd 2024-09-21T09:21:04.859 DEBUG:teuthology.orchestra.run.smithi078:> rpm -q ceph-common --qf '%{VERSION}-%{RELEASE}' 2024-09-21T09:21:04.880 INFO:teuthology.orchestra.run.smithi078.stdout:18.2.4-871.g6a1bdf85.el9 2024-09-21T09:21:04.880 INFO:teuthology.packaging:The installed version of ceph-common is 18.2.4-871.g6a1bdf85.el9 2024-09-21T09:21:04.880 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:21:05.037 INFO:teuthology.task.install:Ceph rpm upgrade from 18.2.4-871.g6a1bdf85.el9 to 19.3.0-5092.g62da54b4 2024-09-21T09:21:05.037 INFO:teuthology.task.install.rpm:Host smithi078 is: centos 9.stream x86_64 2024-09-21T09:21:05.037 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:21:05.182 INFO:teuthology.task.install.rpm:Repo base URL: https://3.chacra.ceph.com/r/ceph/main/62da54b4b83a96ca9976b7e1d0c8272a564a2208/centos/9/flavors/default/ 2024-09-21T09:21:05.182 DEBUG:teuthology.orchestra.run.smithi078:> sudo rm -f /etc/yum.repos.d/ceph.repo 2024-09-21T09:21:05.318 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://3.chacra.ceph.com/r/ceph/main/62da54b4b83a96ca9976b7e1d0c8272a564a2208/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://3.chacra.ceph.com/r/ceph/main/62da54b4b83a96ca9976b7e1d0c8272a564a2208/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://3.chacra.ceph.com/r/ceph/main/62da54b4b83a96ca9976b7e1d0c8272a564a2208/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-09-21T09:21:05.318 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-21T09:21:05.318 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-09-21T09:21:05.344 DEBUG:teuthology.orchestra.run.smithi078:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;sha1/62da54b4b83a96ca9976b7e1d0c8272a564a2208/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-09-21T09:21:05.410 DEBUG:teuthology.orchestra.run.smithi078:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-09-21T09:21:05.475 DEBUG:teuthology.orchestra.run.smithi078:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-09-21T09:21:05.530 INFO:teuthology.orchestra.run.smithi078.stdout:check_obsoletes = 1 2024-09-21T09:21:05.542 DEBUG:teuthology.orchestra.run.smithi078:> sudo yum clean all 2024-09-21T09:21:05.772 INFO:teuthology.orchestra.run.smithi078.stdout:60 files removed 2024-09-21T09:21:05.800 DEBUG:teuthology.orchestra.run.smithi078:> sudo yum -y upgrade ceph-radosgw ceph-test ceph ceph-base cephadm ceph-immutable-object-cache ceph-mgr ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-rook ceph-mgr-cephadm ceph-fuse ceph-volume librados-devel libcephfs2 libcephfs-devel librados2 librbd1 python3-rados python3-rgw python3-cephfs python3-rbd rbd-fuse rbd-mirror rbd-nbd 2024-09-21T09:21:06.359 INFO:teuthology.orchestra.run.smithi078.stdout:ceph packages for x86_64 313 kB/s | 84 kB 00:00 2024-09-21T09:21:06.676 INFO:teuthology.orchestra.run.smithi078.stdout:ceph noarch packages 88 kB/s | 24 kB 00:00 2024-09-21T09:21:06.952 INFO:teuthology.orchestra.run.smithi078.stdout:ceph source packages 8.1 kB/s | 1.9 kB 00:00 2024-09-21T09:21:08.777 INFO:teuthology.orchestra.run.smithi078.stdout:CentOS Stream 9 - BaseOS 4.6 MB/s | 8.3 MB 00:01 2024-09-21T09:21:13.228 INFO:teuthology.orchestra.run.smithi078.stdout:CentOS Stream 9 - AppStream 6.0 MB/s | 20 MB 00:03 2024-09-21T09:21:18.822 INFO:teuthology.orchestra.run.smithi078.stdout:CentOS Stream 9 - CRB 3.7 MB/s | 6.5 MB 00:01 2024-09-21T09:21:21.163 INFO:teuthology.orchestra.run.smithi078.stdout:CentOS Stream 9 - Extras packages 17 kB/s | 19 kB 00:01 2024-09-21T09:21:21.960 INFO:teuthology.orchestra.run.smithi078.stdout:Extra Packages for Enterprise Linux 34 MB/s | 23 MB 00:00 2024-09-21T09:21:27.340 INFO:teuthology.orchestra.run.smithi078.stdout:lab-extras 29 kB/s | 1.7 kB 00:00 2024-09-21T09:21:29.565 INFO:teuthology.orchestra.run.smithi078.stderr:Package cephadm available, but not installed. 2024-09-21T09:21:29.565 INFO:teuthology.orchestra.run.smithi078.stdout:No match for argument: cephadm 2024-09-21T09:21:29.571 INFO:teuthology.orchestra.run.smithi078.stderr:Package ceph-mgr-dashboard available, but not installed. 2024-09-21T09:21:29.571 INFO:teuthology.orchestra.run.smithi078.stdout:No match for argument: ceph-mgr-dashboard 2024-09-21T09:21:29.572 INFO:teuthology.orchestra.run.smithi078.stderr:Package ceph-mgr-diskprediction-local available, but not installed. 2024-09-21T09:21:29.572 INFO:teuthology.orchestra.run.smithi078.stdout:No match for argument: ceph-mgr-diskprediction-local 2024-09-21T09:21:29.572 INFO:teuthology.orchestra.run.smithi078.stderr:Package ceph-mgr-rook available, but not installed. 2024-09-21T09:21:29.572 INFO:teuthology.orchestra.run.smithi078.stdout:No match for argument: ceph-mgr-rook 2024-09-21T09:21:29.573 INFO:teuthology.orchestra.run.smithi078.stderr:Package ceph-mgr-cephadm available, but not installed. 2024-09-21T09:21:29.573 INFO:teuthology.orchestra.run.smithi078.stdout:No match for argument: ceph-mgr-cephadm 2024-09-21T09:21:29.575 INFO:teuthology.orchestra.run.smithi078.stderr:Package ceph-volume available, but not installed. 2024-09-21T09:21:29.575 INFO:teuthology.orchestra.run.smithi078.stdout:No match for argument: ceph-volume 2024-09-21T09:21:29.664 INFO:teuthology.orchestra.run.smithi078.stdout:Dependencies resolved. 2024-09-21T09:21:29.667 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-21T09:21:29.667 INFO:teuthology.orchestra.run.smithi078.stdout: Package Arch Version Repository Size 2024-09-21T09:21:29.667 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-21T09:21:29.667 INFO:teuthology.orchestra.run.smithi078.stdout:Upgrading: 2024-09-21T09:21:29.667 INFO:teuthology.orchestra.run.smithi078.stdout: ceph x86_64 2:19.3.0-5092.g62da54b4.el9 ceph 6.6 k 2024-09-21T09:21:29.667 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-base x86_64 2:19.3.0-5092.g62da54b4.el9 ceph 5.5 M 2024-09-21T09:21:29.668 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-common x86_64 2:19.3.0-5092.g62da54b4.el9 ceph 22 M 2024-09-21T09:21:29.668 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-fuse x86_64 2:19.3.0-5092.g62da54b4.el9 ceph 900 k 2024-09-21T09:21:29.668 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-immutable-object-cache 2024-09-21T09:21:29.668 INFO:teuthology.orchestra.run.smithi078.stdout: x86_64 2:19.3.0-5092.g62da54b4.el9 ceph 145 k 2024-09-21T09:21:29.668 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-mds x86_64 2:19.3.0-5092.g62da54b4.el9 ceph 2.4 M 2024-09-21T09:21:29.668 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-mgr x86_64 2:19.3.0-5092.g62da54b4.el9 ceph 1.5 M 2024-09-21T09:21:29.668 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-mgr-modules-core noarch 2:19.3.0-5092.g62da54b4.el9 ceph-noarch 292 k 2024-09-21T09:21:29.668 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-mon x86_64 2:19.3.0-5092.g62da54b4.el9 ceph 4.8 M 2024-09-21T09:21:29.668 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-osd x86_64 2:19.3.0-5092.g62da54b4.el9 ceph 17 M 2024-09-21T09:21:29.668 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-radosgw x86_64 2:19.3.0-5092.g62da54b4.el9 ceph 11 M 2024-09-21T09:21:29.668 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-selinux x86_64 2:19.3.0-5092.g62da54b4.el9 ceph 25 k 2024-09-21T09:21:29.668 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-test x86_64 2:19.3.0-5092.g62da54b4.el9 ceph 46 M 2024-09-21T09:21:29.668 INFO:teuthology.orchestra.run.smithi078.stdout: libcephfs-devel x86_64 2:19.3.0-5092.g62da54b4.el9 ceph 32 k 2024-09-21T09:21:29.668 INFO:teuthology.orchestra.run.smithi078.stdout: libcephfs2 x86_64 2:19.3.0-5092.g62da54b4.el9 ceph 756 k 2024-09-21T09:21:29.668 INFO:teuthology.orchestra.run.smithi078.stdout: libcephsqlite x86_64 2:19.3.0-5092.g62da54b4.el9 ceph 161 k 2024-09-21T09:21:29.669 INFO:teuthology.orchestra.run.smithi078.stdout: librados-devel x86_64 2:19.3.0-5092.g62da54b4.el9 ceph 126 k 2024-09-21T09:21:29.669 INFO:teuthology.orchestra.run.smithi078.stdout: librados2 x86_64 2:19.3.0-5092.g62da54b4.el9 ceph 3.5 M 2024-09-21T09:21:29.669 INFO:teuthology.orchestra.run.smithi078.stdout: libradosstriper1 x86_64 2:19.3.0-5092.g62da54b4.el9 ceph 508 k 2024-09-21T09:21:29.669 INFO:teuthology.orchestra.run.smithi078.stdout: librbd1 x86_64 2:19.3.0-5092.g62da54b4.el9 ceph 3.2 M 2024-09-21T09:21:29.669 INFO:teuthology.orchestra.run.smithi078.stdout: librgw2 x86_64 2:19.3.0-5092.g62da54b4.el9 ceph 5.5 M 2024-09-21T09:21:29.669 INFO:teuthology.orchestra.run.smithi078.stdout: python3-ceph-argparse x86_64 2:19.3.0-5092.g62da54b4.el9 ceph 45 k 2024-09-21T09:21:29.669 INFO:teuthology.orchestra.run.smithi078.stdout: python3-ceph-common x86_64 2:19.3.0-5092.g62da54b4.el9 ceph 148 k 2024-09-21T09:21:29.669 INFO:teuthology.orchestra.run.smithi078.stdout: python3-cephfs x86_64 2:19.3.0-5092.g62da54b4.el9 ceph 161 k 2024-09-21T09:21:29.669 INFO:teuthology.orchestra.run.smithi078.stdout: python3-rados x86_64 2:19.3.0-5092.g62da54b4.el9 ceph 321 k 2024-09-21T09:21:29.669 INFO:teuthology.orchestra.run.smithi078.stdout: python3-rbd x86_64 2:19.3.0-5092.g62da54b4.el9 ceph 305 k 2024-09-21T09:21:29.669 INFO:teuthology.orchestra.run.smithi078.stdout: python3-rgw x86_64 2:19.3.0-5092.g62da54b4.el9 ceph 99 k 2024-09-21T09:21:29.669 INFO:teuthology.orchestra.run.smithi078.stdout: rbd-fuse x86_64 2:19.3.0-5092.g62da54b4.el9 ceph 85 k 2024-09-21T09:21:29.669 INFO:teuthology.orchestra.run.smithi078.stdout: rbd-mirror x86_64 2:19.3.0-5092.g62da54b4.el9 ceph 3.2 M 2024-09-21T09:21:29.669 INFO:teuthology.orchestra.run.smithi078.stdout: rbd-nbd x86_64 2:19.3.0-5092.g62da54b4.el9 ceph 170 k 2024-09-21T09:21:29.670 INFO:teuthology.orchestra.run.smithi078.stdout:Installing dependencies: 2024-09-21T09:21:29.670 INFO:teuthology.orchestra.run.smithi078.stdout: libnbd x86_64 1.20.2-2.el9 appstream 171 k 2024-09-21T09:21:29.670 INFO:teuthology.orchestra.run.smithi078.stdout: lua x86_64 5.4.4-4.el9 appstream 188 k 2024-09-21T09:21:29.670 INFO:teuthology.orchestra.run.smithi078.stdout: lua-devel x86_64 5.4.4-4.el9 crb 22 k 2024-09-21T09:21:29.670 INFO:teuthology.orchestra.run.smithi078.stdout: luarocks noarch 3.9.2-5.el9 epel 151 k 2024-09-21T09:21:29.670 INFO:teuthology.orchestra.run.smithi078.stdout: python3-packaging noarch 20.9-5.el9 appstream 77 k 2024-09-21T09:21:29.670 INFO:teuthology.orchestra.run.smithi078.stdout: qatlib x86_64 24.02.0-1.el9 appstream 221 k 2024-09-21T09:21:29.670 INFO:teuthology.orchestra.run.smithi078.stdout: qatzip-libs x86_64 1.2.0-1.el9 appstream 47 k 2024-09-21T09:21:29.670 INFO:teuthology.orchestra.run.smithi078.stdout:Installing weak dependencies: 2024-09-21T09:21:29.670 INFO:teuthology.orchestra.run.smithi078.stdout: qatlib-service x86_64 24.02.0-1.el9 appstream 36 k 2024-09-21T09:21:29.670 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:21:29.670 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction Summary 2024-09-21T09:21:29.670 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-21T09:21:29.670 INFO:teuthology.orchestra.run.smithi078.stdout:Install 8 Packages 2024-09-21T09:21:29.670 INFO:teuthology.orchestra.run.smithi078.stdout:Upgrade 30 Packages 2024-09-21T09:21:29.671 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:21:29.671 INFO:teuthology.orchestra.run.smithi078.stdout:Total download size: 130 M 2024-09-21T09:21:29.671 INFO:teuthology.orchestra.run.smithi078.stdout:Downloading Packages: 2024-09-21T09:21:29.884 INFO:teuthology.orchestra.run.smithi078.stdout:(1/38): python3-packaging-20.9-5.el9.noarch.rpm 658 kB/s | 77 kB 00:00 2024-09-21T09:21:29.909 INFO:teuthology.orchestra.run.smithi078.stdout:(2/38): libnbd-1.20.2-2.el9.x86_64.rpm 1.2 MB/s | 171 kB 00:00 2024-09-21T09:21:29.951 INFO:teuthology.orchestra.run.smithi078.stdout:(3/38): lua-5.4.4-4.el9.x86_64.rpm 1.0 MB/s | 188 kB 00:00 2024-09-21T09:21:29.985 INFO:teuthology.orchestra.run.smithi078.stdout:(4/38): qatlib-24.02.0-1.el9.x86_64.rpm 2.1 MB/s | 221 kB 00:00 2024-09-21T09:21:30.010 INFO:teuthology.orchestra.run.smithi078.stdout:(5/38): qatlib-service-24.02.0-1.el9.x86_64.rpm 357 kB/s | 36 kB 00:00 2024-09-21T09:21:30.035 INFO:teuthology.orchestra.run.smithi078.stdout:(6/38): qatzip-libs-1.2.0-1.el9.x86_64.rpm 565 kB/s | 47 kB 00:00 2024-09-21T09:21:30.077 INFO:teuthology.orchestra.run.smithi078.stdout:(7/38): luarocks-3.9.2-5.el9.noarch.rpm 2.2 MB/s | 151 kB 00:00 2024-09-21T09:21:30.119 INFO:teuthology.orchestra.run.smithi078.stdout:(8/38): lua-devel-5.4.4-4.el9.x86_64.rpm 166 kB/s | 22 kB 00:00 2024-09-21T09:21:30.186 INFO:teuthology.orchestra.run.smithi078.stdout:(9/38): ceph-19.3.0-5092.g62da54b4.el9.x86_64.r 44 kB/s | 6.6 kB 00:00 2024-09-21T09:21:30.370 INFO:teuthology.orchestra.run.smithi078.stdout:(10/38): ceph-fuse-19.3.0-5092.g62da54b4.el9.x8 4.8 MB/s | 900 kB 00:00 2024-09-21T09:21:30.437 INFO:teuthology.orchestra.run.smithi078.stdout:(11/38): ceph-immutable-object-cache-19.3.0-509 2.1 MB/s | 145 kB 00:00 2024-09-21T09:21:30.579 INFO:teuthology.orchestra.run.smithi078.stdout:(12/38): ceph-base-19.3.0-5092.g62da54b4.el9.x8 11 MB/s | 5.5 MB 00:00 2024-09-21T09:21:30.637 INFO:teuthology.orchestra.run.smithi078.stdout:(13/38): ceph-mds-19.3.0-5092.g62da54b4.el9.x86 12 MB/s | 2.4 MB 00:00 2024-09-21T09:21:30.712 INFO:teuthology.orchestra.run.smithi078.stdout:(14/38): ceph-mgr-19.3.0-5092.g62da54b4.el9.x86 11 MB/s | 1.5 MB 00:00 2024-09-21T09:21:30.988 INFO:teuthology.orchestra.run.smithi078.stdout:(15/38): ceph-mon-19.3.0-5092.g62da54b4.el9.x86 14 MB/s | 4.8 MB 00:00 2024-09-21T09:21:31.463 INFO:teuthology.orchestra.run.smithi078.stdout:(16/38): ceph-common-19.3.0-5092.g62da54b4.el9. 16 MB/s | 22 MB 00:01 2024-09-21T09:21:31.505 INFO:teuthology.orchestra.run.smithi078.stdout:(17/38): ceph-selinux-19.3.0-5092.g62da54b4.el9 608 kB/s | 25 kB 00:00 2024-09-21T09:21:31.680 INFO:teuthology.orchestra.run.smithi078.stdout:(18/38): ceph-radosgw-19.3.0-5092.g62da54b4.el9 16 MB/s | 11 MB 00:00 2024-09-21T09:21:31.847 INFO:teuthology.orchestra.run.smithi078.stdout:(19/38): ceph-osd-19.3.0-5092.g62da54b4.el9.x86 15 MB/s | 17 MB 00:01 2024-09-21T09:21:31.872 INFO:teuthology.orchestra.run.smithi078.stdout:(20/38): libcephfs-devel-19.3.0-5092.g62da54b4. 166 kB/s | 32 kB 00:00 2024-09-21T09:21:31.915 INFO:teuthology.orchestra.run.smithi078.stdout:(21/38): libcephfs2-19.3.0-5092.g62da54b4.el9.x 11 MB/s | 756 kB 00:00 2024-09-21T09:21:31.940 INFO:teuthology.orchestra.run.smithi078.stdout:(22/38): libcephsqlite-19.3.0-5092.g62da54b4.el 2.4 MB/s | 161 kB 00:00 2024-09-21T09:21:31.965 INFO:teuthology.orchestra.run.smithi078.stdout:(23/38): librados-devel-19.3.0-5092.g62da54b4.e 2.5 MB/s | 126 kB 00:00 2024-09-21T09:21:32.032 INFO:teuthology.orchestra.run.smithi078.stdout:(24/38): libradosstriper1-19.3.0-5092.g62da54b4 7.4 MB/s | 508 kB 00:00 2024-09-21T09:21:32.116 INFO:teuthology.orchestra.run.smithi078.stdout:(25/38): librados2-19.3.0-5092.g62da54b4.el9.x8 20 MB/s | 3.5 MB 00:00 2024-09-21T09:21:32.283 INFO:teuthology.orchestra.run.smithi078.stdout:(26/38): librbd1-19.3.0-5092.g62da54b4.el9.x86_ 13 MB/s | 3.2 MB 00:00 2024-09-21T09:21:32.350 INFO:teuthology.orchestra.run.smithi078.stdout:(27/38): python3-ceph-argparse-19.3.0-5092.g62d 677 kB/s | 45 kB 00:00 2024-09-21T09:21:32.433 INFO:teuthology.orchestra.run.smithi078.stdout:(28/38): librgw2-19.3.0-5092.g62da54b4.el9.x86_ 17 MB/s | 5.5 MB 00:00 2024-09-21T09:21:32.459 INFO:teuthology.orchestra.run.smithi078.stdout:(29/38): python3-ceph-common-19.3.0-5092.g62da5 1.3 MB/s | 148 kB 00:00 2024-09-21T09:21:32.484 INFO:teuthology.orchestra.run.smithi078.stdout:(30/38): python3-cephfs-19.3.0-5092.g62da54b4.e 3.1 MB/s | 161 kB 00:00 2024-09-21T09:21:32.517 INFO:teuthology.orchestra.run.smithi078.stdout:(31/38): python3-rados-19.3.0-5092.g62da54b4.el 5.4 MB/s | 321 kB 00:00 2024-09-21T09:21:32.543 INFO:teuthology.orchestra.run.smithi078.stdout:(32/38): python3-rbd-19.3.0-5092.g62da54b4.el9. 5.1 MB/s | 305 kB 00:00 2024-09-21T09:21:32.568 INFO:teuthology.orchestra.run.smithi078.stdout:(33/38): python3-rgw-19.3.0-5092.g62da54b4.el9. 1.9 MB/s | 99 kB 00:00 2024-09-21T09:21:32.593 INFO:teuthology.orchestra.run.smithi078.stdout:(34/38): rbd-fuse-19.3.0-5092.g62da54b4.el9.x86 1.7 MB/s | 85 kB 00:00 2024-09-21T09:21:32.652 INFO:teuthology.orchestra.run.smithi078.stdout:(35/38): rbd-nbd-19.3.0-5092.g62da54b4.el9.x86_ 2.8 MB/s | 170 kB 00:00 2024-09-21T09:21:32.711 INFO:teuthology.orchestra.run.smithi078.stdout:(36/38): rbd-mirror-19.3.0-5092.g62da54b4.el9.x 22 MB/s | 3.2 MB 00:00 2024-09-21T09:21:32.736 INFO:teuthology.orchestra.run.smithi078.stdout:(37/38): ceph-mgr-modules-core-19.3.0-5092.g62d 3.4 MB/s | 292 kB 00:00 2024-09-21T09:21:33.512 INFO:teuthology.orchestra.run.smithi078.stdout:(38/38): ceph-test-19.3.0-5092.g62da54b4.el9.x8 23 MB/s | 46 MB 00:02 2024-09-21T09:21:33.516 INFO:teuthology.orchestra.run.smithi078.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:21:33.516 INFO:teuthology.orchestra.run.smithi078.stdout:Total 34 MB/s | 130 MB 00:03 2024-09-21T09:21:33.599 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction check 2024-09-21T09:21:33.655 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction check succeeded. 2024-09-21T09:21:33.655 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction test 2024-09-21T09:21:34.607 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction test succeeded. 2024-09-21T09:21:34.607 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction 2024-09-21T09:21:37.388 INFO:teuthology.orchestra.run.smithi078.stdout: Preparing : 1/1 2024-09-21T09:21:37.495 INFO:teuthology.orchestra.run.smithi078.stdout: Upgrading : librados2-2:19.3.0-5092.g62da54b4.el9.x86_64 1/68 2024-09-21T09:21:37.619 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: librados2-2:19.3.0-5092.g62da54b4.el9.x86_64 1/68 2024-09-21T09:21:37.680 INFO:teuthology.orchestra.run.smithi078.stdout: Upgrading : libcephfs2-2:19.3.0-5092.g62da54b4.el9.x86_64 2/68 2024-09-21T09:21:38.315 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: libcephfs2-2:19.3.0-5092.g62da54b4.el9.x86_64 2/68 2024-09-21T09:21:38.373 INFO:teuthology.orchestra.run.smithi078.stdout: Upgrading : librgw2-2:19.3.0-5092.g62da54b4.el9.x86_64 3/68 2024-09-21T09:21:38.464 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: librgw2-2:19.3.0-5092.g62da54b4.el9.x86_64 3/68 2024-09-21T09:21:38.507 INFO:teuthology.orchestra.run.smithi078.stdout: Upgrading : python3-rados-2:19.3.0-5092.g62da54b4.el9.x86_64 4/68 2024-09-21T09:21:38.830 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : libnbd-1.20.2-2.el9.x86_64 5/68 2024-09-21T09:21:38.892 INFO:teuthology.orchestra.run.smithi078.stdout: Upgrading : librbd1-2:19.3.0-5092.g62da54b4.el9.x86_64 6/68 2024-09-21T09:21:38.950 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: librbd1-2:19.3.0-5092.g62da54b4.el9.x86_64 6/68 2024-09-21T09:21:38.977 INFO:teuthology.orchestra.run.smithi078.stdout: Upgrading : libcephsqlite-2:19.3.0-5092.g62da54b4.el9.x86_64 7/68 2024-09-21T09:21:39.082 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: libcephsqlite-2:19.3.0-5092.g62da54b4.el9.x86_64 7/68 2024-09-21T09:21:39.128 INFO:teuthology.orchestra.run.smithi078.stdout: Upgrading : libradosstriper1-2:19.3.0-5092.g62da54b4.el9.x86_6 8/68 2024-09-21T09:21:39.191 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: libradosstriper1-2:19.3.0-5092.g62da54b4.el9.x86_6 8/68 2024-09-21T09:21:39.229 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : qatlib-service-24.02.0-1.el9.x86_64 9/68 2024-09-21T09:21:39.245 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: qatlib-service-24.02.0-1.el9.x86_64 9/68 2024-09-21T09:21:39.430 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: qatlib-24.02.0-1.el9.x86_64 10/68 2024-09-21T09:21:39.518 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : qatlib-24.02.0-1.el9.x86_64 10/68 2024-09-21T09:21:39.558 INFO:teuthology.orchestra.run.smithi078.stdout: Upgrading : python3-ceph-argparse-2:19.3.0-5092.g62da54b4.el9. 11/68 2024-09-21T09:21:39.595 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : lua-5.4.4-4.el9.x86_64 12/68 2024-09-21T09:21:39.673 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : lua-devel-5.4.4-4.el9.x86_64 13/68 2024-09-21T09:21:39.730 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : luarocks-3.9.2-5.el9.noarch 14/68 2024-09-21T09:21:39.781 INFO:teuthology.orchestra.run.smithi078.stdout: Upgrading : python3-cephfs-2:19.3.0-5092.g62da54b4.el9.x86_64 15/68 2024-09-21T09:21:39.834 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : qatzip-libs-1.2.0-1.el9.x86_64 16/68 2024-09-21T09:21:39.889 INFO:teuthology.orchestra.run.smithi078.stdout: Upgrading : python3-rbd-2:19.3.0-5092.g62da54b4.el9.x86_64 17/68 2024-09-21T09:21:39.934 INFO:teuthology.orchestra.run.smithi078.stdout: Upgrading : python3-rgw-2:19.3.0-5092.g62da54b4.el9.x86_64 18/68 2024-09-21T09:21:40.058 INFO:teuthology.orchestra.run.smithi078.stdout: Upgrading : librados-devel-2:19.3.0-5092.g62da54b4.el9.x86_64 19/68 2024-09-21T09:21:40.106 INFO:teuthology.orchestra.run.smithi078.stdout: Upgrading : python3-ceph-common-2:19.3.0-5092.g62da54b4.el9.x8 20/68 2024-09-21T09:21:42.625 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: ceph-common-2:19.3.0-5092.g62da54b4.el9.x86_64 21/68 2024-09-21T09:21:42.732 INFO:teuthology.orchestra.run.smithi078.stdout: Upgrading : ceph-common-2:19.3.0-5092.g62da54b4.el9.x86_64 21/68 2024-09-21T09:21:43.407 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: ceph-common-2:19.3.0-5092.g62da54b4.el9.x86_64 21/68 2024-09-21T09:21:43.552 INFO:teuthology.orchestra.run.smithi078.stdout: Upgrading : ceph-base-2:19.3.0-5092.g62da54b4.el9.x86_64 22/68 2024-09-21T09:21:43.617 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: ceph-base-2:19.3.0-5092.g62da54b4.el9.x86_64 22/68 2024-09-21T09:21:43.654 INFO:teuthology.orchestra.run.smithi078.stdout: Upgrading : ceph-selinux-2:19.3.0-5092.g62da54b4.el9.x86_64 23/68 2024-09-21T09:21:52.190 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: ceph-selinux-2:19.3.0-5092.g62da54b4.el9.x86_64 23/68 2024-09-21T09:21:52.260 INFO:teuthology.orchestra.run.smithi078.stdout: Upgrading : ceph-mds-2:19.3.0-5092.g62da54b4.el9.x86_64 24/68 2024-09-21T09:21:52.820 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: ceph-mds-2:19.3.0-5092.g62da54b4.el9.x86_64 24/68 2024-09-21T09:21:52.869 INFO:teuthology.orchestra.run.smithi078.stdout: Upgrading : ceph-mon-2:19.3.0-5092.g62da54b4.el9.x86_64 25/68 2024-09-21T09:21:54.732 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: ceph-mon-2:19.3.0-5092.g62da54b4.el9.x86_64 25/68 2024-09-21T09:21:54.797 INFO:teuthology.orchestra.run.smithi078.stdout: Upgrading : ceph-osd-2:19.3.0-5092.g62da54b4.el9.x86_64 26/68 2024-09-21T09:21:54.832 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: ceph-osd-2:19.3.0-5092.g62da54b4.el9.x86_64 26/68 2024-09-21T09:21:54.989 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-packaging-20.9-5.el9.noarch 27/68 2024-09-21T09:21:55.187 INFO:teuthology.orchestra.run.smithi078.stdout: Upgrading : ceph-mgr-modules-core-2:19.3.0-5092.g62da54b4.el9. 28/68 2024-09-21T09:21:55.267 INFO:teuthology.orchestra.run.smithi078.stdout: Upgrading : ceph-mgr-2:19.3.0-5092.g62da54b4.el9.x86_64 29/68 2024-09-21T09:21:55.273 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: ceph-mgr-2:19.3.0-5092.g62da54b4.el9.x86_64 29/68 2024-09-21T09:21:56.407 INFO:teuthology.orchestra.run.smithi078.stdout: Upgrading : ceph-2:19.3.0-5092.g62da54b4.el9.x86_64 30/68 2024-09-21T09:21:56.535 INFO:teuthology.orchestra.run.smithi078.stdout: Upgrading : ceph-radosgw-2:19.3.0-5092.g62da54b4.el9.x86_64 31/68 2024-09-21T09:21:56.556 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: ceph-radosgw-2:19.3.0-5092.g62da54b4.el9.x86_64 31/68 2024-09-21T09:21:56.594 INFO:teuthology.orchestra.run.smithi078.stdout: Upgrading : ceph-immutable-object-cache-2:19.3.0-5092.g62da54b 32/68 2024-09-21T09:21:56.973 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: ceph-immutable-object-cache-2:19.3.0-5092.g62da54b 32/68 2024-09-21T09:21:57.011 INFO:teuthology.orchestra.run.smithi078.stdout: Upgrading : rbd-mirror-2:19.3.0-5092.g62da54b4.el9.x86_64 33/68 2024-09-21T09:22:02.085 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: rbd-mirror-2:19.3.0-5092.g62da54b4.el9.x86_64 33/68 2024-09-21T09:22:02.526 INFO:teuthology.orchestra.run.smithi078.stdout: Upgrading : ceph-test-2:19.3.0-5092.g62da54b4.el9.x86_64 34/68 2024-09-21T09:22:02.579 INFO:teuthology.orchestra.run.smithi078.stdout: Upgrading : libcephfs-devel-2:19.3.0-5092.g62da54b4.el9.x86_64 35/68 2024-09-21T09:22:02.628 INFO:teuthology.orchestra.run.smithi078.stdout: Upgrading : rbd-fuse-2:19.3.0-5092.g62da54b4.el9.x86_64 36/68 2024-09-21T09:22:02.749 INFO:teuthology.orchestra.run.smithi078.stdout: Upgrading : rbd-nbd-2:19.3.0-5092.g62da54b4.el9.x86_64 37/68 2024-09-21T09:22:02.806 INFO:teuthology.orchestra.run.smithi078.stdout: Upgrading : ceph-fuse-2:19.3.0-5092.g62da54b4.el9.x86_64 38/68 2024-09-21T09:22:02.829 INFO:teuthology.orchestra.run.smithi078.stdout: Cleanup : ceph-2:18.2.4-871.g6a1bdf85.el9.x86_64 39/68 2024-09-21T09:22:02.868 INFO:teuthology.orchestra.run.smithi078.stdout: Cleanup : ceph-test-2:18.2.4-871.g6a1bdf85.el9.x86_64 40/68 2024-09-21T09:22:02.875 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: ceph-radosgw-2:18.2.4-871.g6a1bdf85.el9.x86_64 41/68 2024-09-21T09:22:02.875 INFO:teuthology.orchestra.run.smithi078.stdout: Cleanup : ceph-radosgw-2:18.2.4-871.g6a1bdf85.el9.x86_64 41/68 2024-09-21T09:22:02.928 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: ceph-radosgw-2:18.2.4-871.g6a1bdf85.el9.x86_64 41/68 2024-09-21T09:22:02.952 INFO:teuthology.orchestra.run.smithi078.stdout: Cleanup : rbd-nbd-2:18.2.4-871.g6a1bdf85.el9.x86_64 42/68 2024-09-21T09:22:02.958 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: rbd-mirror-2:18.2.4-871.g6a1bdf85.el9.x86_64 43/68 2024-09-21T09:22:02.958 INFO:teuthology.orchestra.run.smithi078.stdout: Cleanup : rbd-mirror-2:18.2.4-871.g6a1bdf85.el9.x86_64 43/68 2024-09-21T09:22:02.986 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: rbd-mirror-2:18.2.4-871.g6a1bdf85.el9.x86_64 43/68 2024-09-21T09:22:02.991 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: ceph-mgr-2:18.2.4-871.g6a1bdf85.el9.x86_64 44/68 2024-09-21T09:22:02.991 INFO:teuthology.orchestra.run.smithi078.stdout: Cleanup : ceph-mgr-2:18.2.4-871.g6a1bdf85.el9.x86_64 44/68 2024-09-21T09:22:03.020 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: ceph-mgr-2:18.2.4-871.g6a1bdf85.el9.x86_64 44/68 2024-09-21T09:22:03.045 INFO:teuthology.orchestra.run.smithi078.stdout: Cleanup : rbd-fuse-2:18.2.4-871.g6a1bdf85.el9.x86_64 45/68 2024-09-21T09:22:03.050 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: ceph-immutable-object-cache-2:18.2.4-871.g6a1bdf85 46/68 2024-09-21T09:22:03.050 INFO:teuthology.orchestra.run.smithi078.stdout: Cleanup : ceph-immutable-object-cache-2:18.2.4-871.g6a1bdf85 46/68 2024-09-21T09:22:03.087 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: ceph-immutable-object-cache-2:18.2.4-871.g6a1bdf85 46/68 2024-09-21T09:22:03.112 INFO:teuthology.orchestra.run.smithi078.stdout: Cleanup : libcephfs-devel-2:18.2.4-871.g6a1bdf85.el9.x86_64 47/68 2024-09-21T09:22:03.112 INFO:teuthology.orchestra.run.smithi078.stdout: Cleanup : libcephsqlite-2:18.2.4-871.g6a1bdf85.el9.x86_64 48/68 2024-09-21T09:22:03.197 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: libcephsqlite-2:18.2.4-871.g6a1bdf85.el9.x86_64 48/68 2024-09-21T09:22:03.222 INFO:teuthology.orchestra.run.smithi078.stdout: Cleanup : librados-devel-2:18.2.4-871.g6a1bdf85.el9.x86_64 49/68 2024-09-21T09:22:03.227 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: ceph-mds-2:18.2.4-871.g6a1bdf85.el9.x86_64 50/68 2024-09-21T09:22:03.227 INFO:teuthology.orchestra.run.smithi078.stdout: Cleanup : ceph-mds-2:18.2.4-871.g6a1bdf85.el9.x86_64 50/68 2024-09-21T09:22:03.264 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: ceph-mds-2:18.2.4-871.g6a1bdf85.el9.x86_64 50/68 2024-09-21T09:22:03.269 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: ceph-mon-2:18.2.4-871.g6a1bdf85.el9.x86_64 51/68 2024-09-21T09:22:03.269 INFO:teuthology.orchestra.run.smithi078.stdout: Cleanup : ceph-mon-2:18.2.4-871.g6a1bdf85.el9.x86_64 51/68 2024-09-21T09:22:03.298 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: ceph-mon-2:18.2.4-871.g6a1bdf85.el9.x86_64 51/68 2024-09-21T09:22:03.303 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: ceph-osd-2:18.2.4-871.g6a1bdf85.el9.x86_64 52/68 2024-09-21T09:22:03.303 INFO:teuthology.orchestra.run.smithi078.stdout: Cleanup : ceph-osd-2:18.2.4-871.g6a1bdf85.el9.x86_64 52/68 2024-09-21T09:22:03.340 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: ceph-osd-2:18.2.4-871.g6a1bdf85.el9.x86_64 52/68 2024-09-21T09:22:03.340 INFO:teuthology.orchestra.run.smithi078.stdout: Cleanup : ceph-selinux-2:18.2.4-871.g6a1bdf85.el9.x86_64 53/68 2024-09-21T09:22:03.374 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: ceph-selinux-2:18.2.4-871.g6a1bdf85.el9.x86_64 53/68 2024-09-21T09:22:03.380 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: ceph-base-2:18.2.4-871.g6a1bdf85.el9.x86_64 54/68 2024-09-21T09:22:03.380 INFO:teuthology.orchestra.run.smithi078.stdout: Cleanup : ceph-base-2:18.2.4-871.g6a1bdf85.el9.x86_64 54/68 2024-09-21T09:22:03.486 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: ceph-base-2:18.2.4-871.g6a1bdf85.el9.x86_64 54/68 2024-09-21T09:22:03.486 INFO:teuthology.orchestra.run.smithi078.stdout: Cleanup : ceph-common-2:18.2.4-871.g6a1bdf85.el9.x86_64 55/68 2024-09-21T09:22:03.528 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: ceph-common-2:18.2.4-871.g6a1bdf85.el9.x86_64 55/68 2024-09-21T09:22:03.528 INFO:teuthology.orchestra.run.smithi078.stdout: Cleanup : libradosstriper1-2:18.2.4-871.g6a1bdf85.el9.x86_64 56/68 2024-09-21T09:22:03.620 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: libradosstriper1-2:18.2.4-871.g6a1bdf85.el9.x86_64 56/68 2024-09-21T09:22:03.645 INFO:teuthology.orchestra.run.smithi078.stdout: Cleanup : python3-cephfs-2:18.2.4-871.g6a1bdf85.el9.x86_64 57/68 2024-09-21T09:22:03.671 INFO:teuthology.orchestra.run.smithi078.stdout: Cleanup : python3-rbd-2:18.2.4-871.g6a1bdf85.el9.x86_64 58/68 2024-09-21T09:22:03.671 INFO:teuthology.orchestra.run.smithi078.stdout: Cleanup : librbd1-2:18.2.4-871.g6a1bdf85.el9.x86_64 59/68 2024-09-21T09:22:03.780 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: librbd1-2:18.2.4-871.g6a1bdf85.el9.x86_64 59/68 2024-09-21T09:22:03.806 INFO:teuthology.orchestra.run.smithi078.stdout: Cleanup : python3-rgw-2:18.2.4-871.g6a1bdf85.el9.x86_64 60/68 2024-09-21T09:22:03.806 INFO:teuthology.orchestra.run.smithi078.stdout: Cleanup : librgw2-2:18.2.4-871.g6a1bdf85.el9.x86_64 61/68 2024-09-21T09:22:03.890 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: librgw2-2:18.2.4-871.g6a1bdf85.el9.x86_64 61/68 2024-09-21T09:22:03.915 INFO:teuthology.orchestra.run.smithi078.stdout: Cleanup : python3-rados-2:18.2.4-871.g6a1bdf85.el9.x86_64 62/68 2024-09-21T09:22:03.915 INFO:teuthology.orchestra.run.smithi078.stdout: Cleanup : libcephfs2-2:18.2.4-871.g6a1bdf85.el9.x86_64 63/68 2024-09-21T09:22:03.999 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: libcephfs2-2:18.2.4-871.g6a1bdf85.el9.x86_64 63/68 2024-09-21T09:22:04.041 INFO:teuthology.orchestra.run.smithi078.stdout: Cleanup : ceph-fuse-2:18.2.4-871.g6a1bdf85.el9.x86_64 64/68 2024-09-21T09:22:04.066 INFO:teuthology.orchestra.run.smithi078.stdout: Cleanup : python3-ceph-argparse-2:18.2.4-871.g6a1bdf85.el9.x 65/68 2024-09-21T09:22:04.091 INFO:teuthology.orchestra.run.smithi078.stdout: Cleanup : python3-ceph-common-2:18.2.4-871.g6a1bdf85.el9.x86 66/68 2024-09-21T09:22:04.117 INFO:teuthology.orchestra.run.smithi078.stdout: Cleanup : ceph-mgr-modules-core-2:18.2.4-871.g6a1bdf85.el9.n 67/68 2024-09-21T09:22:04.117 INFO:teuthology.orchestra.run.smithi078.stdout: Cleanup : librados2-2:18.2.4-871.g6a1bdf85.el9.x86_64 68/68 2024-09-21T09:22:05.661 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: librados2-2:18.2.4-871.g6a1bdf85.el9.x86_64 68/68 2024-09-21T09:22:05.661 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : libnbd-1.20.2-2.el9.x86_64 1/68 2024-09-21T09:22:05.661 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : lua-5.4.4-4.el9.x86_64 2/68 2024-09-21T09:22:05.661 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-packaging-20.9-5.el9.noarch 3/68 2024-09-21T09:22:05.661 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : qatlib-24.02.0-1.el9.x86_64 4/68 2024-09-21T09:22:05.662 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : qatlib-service-24.02.0-1.el9.x86_64 5/68 2024-09-21T09:22:05.662 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : qatzip-libs-1.2.0-1.el9.x86_64 6/68 2024-09-21T09:22:05.662 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : lua-devel-5.4.4-4.el9.x86_64 7/68 2024-09-21T09:22:05.662 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : luarocks-3.9.2-5.el9.noarch 8/68 2024-09-21T09:22:05.662 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : ceph-2:19.3.0-5092.g62da54b4.el9.x86_64 9/68 2024-09-21T09:22:05.662 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : ceph-2:18.2.4-871.g6a1bdf85.el9.x86_64 10/68 2024-09-21T09:22:05.662 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : ceph-base-2:19.3.0-5092.g62da54b4.el9.x86_64 11/68 2024-09-21T09:22:05.662 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : ceph-base-2:18.2.4-871.g6a1bdf85.el9.x86_64 12/68 2024-09-21T09:22:05.662 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : ceph-common-2:19.3.0-5092.g62da54b4.el9.x86_64 13/68 2024-09-21T09:22:05.662 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : ceph-common-2:18.2.4-871.g6a1bdf85.el9.x86_64 14/68 2024-09-21T09:22:05.662 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : ceph-fuse-2:19.3.0-5092.g62da54b4.el9.x86_64 15/68 2024-09-21T09:22:05.662 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : ceph-fuse-2:18.2.4-871.g6a1bdf85.el9.x86_64 16/68 2024-09-21T09:22:05.663 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : ceph-immutable-object-cache-2:19.3.0-5092.g62da54b 17/68 2024-09-21T09:22:05.663 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : ceph-immutable-object-cache-2:18.2.4-871.g6a1bdf85 18/68 2024-09-21T09:22:05.663 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : ceph-mds-2:19.3.0-5092.g62da54b4.el9.x86_64 19/68 2024-09-21T09:22:05.663 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : ceph-mds-2:18.2.4-871.g6a1bdf85.el9.x86_64 20/68 2024-09-21T09:22:05.663 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : ceph-mgr-2:19.3.0-5092.g62da54b4.el9.x86_64 21/68 2024-09-21T09:22:05.663 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : ceph-mgr-2:18.2.4-871.g6a1bdf85.el9.x86_64 22/68 2024-09-21T09:22:05.663 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : ceph-mon-2:19.3.0-5092.g62da54b4.el9.x86_64 23/68 2024-09-21T09:22:05.663 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : ceph-mon-2:18.2.4-871.g6a1bdf85.el9.x86_64 24/68 2024-09-21T09:22:05.663 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : ceph-osd-2:19.3.0-5092.g62da54b4.el9.x86_64 25/68 2024-09-21T09:22:05.663 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : ceph-osd-2:18.2.4-871.g6a1bdf85.el9.x86_64 26/68 2024-09-21T09:22:05.663 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : ceph-radosgw-2:19.3.0-5092.g62da54b4.el9.x86_64 27/68 2024-09-21T09:22:05.663 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : ceph-radosgw-2:18.2.4-871.g6a1bdf85.el9.x86_64 28/68 2024-09-21T09:22:05.663 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : ceph-selinux-2:19.3.0-5092.g62da54b4.el9.x86_64 29/68 2024-09-21T09:22:05.663 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : ceph-selinux-2:18.2.4-871.g6a1bdf85.el9.x86_64 30/68 2024-09-21T09:22:05.664 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : ceph-test-2:19.3.0-5092.g62da54b4.el9.x86_64 31/68 2024-09-21T09:22:05.664 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : ceph-test-2:18.2.4-871.g6a1bdf85.el9.x86_64 32/68 2024-09-21T09:22:05.667 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : libcephfs-devel-2:19.3.0-5092.g62da54b4.el9.x86_64 33/68 2024-09-21T09:22:05.667 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : libcephfs-devel-2:18.2.4-871.g6a1bdf85.el9.x86_64 34/68 2024-09-21T09:22:05.667 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : libcephfs2-2:19.3.0-5092.g62da54b4.el9.x86_64 35/68 2024-09-21T09:22:05.667 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : libcephfs2-2:18.2.4-871.g6a1bdf85.el9.x86_64 36/68 2024-09-21T09:22:05.667 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : libcephsqlite-2:19.3.0-5092.g62da54b4.el9.x86_64 37/68 2024-09-21T09:22:05.667 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : libcephsqlite-2:18.2.4-871.g6a1bdf85.el9.x86_64 38/68 2024-09-21T09:22:05.667 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : librados-devel-2:19.3.0-5092.g62da54b4.el9.x86_64 39/68 2024-09-21T09:22:05.667 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : librados-devel-2:18.2.4-871.g6a1bdf85.el9.x86_64 40/68 2024-09-21T09:22:05.667 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : librados2-2:19.3.0-5092.g62da54b4.el9.x86_64 41/68 2024-09-21T09:22:05.668 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : librados2-2:18.2.4-871.g6a1bdf85.el9.x86_64 42/68 2024-09-21T09:22:05.668 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : libradosstriper1-2:19.3.0-5092.g62da54b4.el9.x86_6 43/68 2024-09-21T09:22:05.668 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : libradosstriper1-2:18.2.4-871.g6a1bdf85.el9.x86_64 44/68 2024-09-21T09:22:05.668 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : librbd1-2:19.3.0-5092.g62da54b4.el9.x86_64 45/68 2024-09-21T09:22:05.668 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : librbd1-2:18.2.4-871.g6a1bdf85.el9.x86_64 46/68 2024-09-21T09:22:05.668 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : librgw2-2:19.3.0-5092.g62da54b4.el9.x86_64 47/68 2024-09-21T09:22:05.668 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : librgw2-2:18.2.4-871.g6a1bdf85.el9.x86_64 48/68 2024-09-21T09:22:05.668 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-ceph-argparse-2:19.3.0-5092.g62da54b4.el9. 49/68 2024-09-21T09:22:05.668 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-ceph-argparse-2:18.2.4-871.g6a1bdf85.el9.x 50/68 2024-09-21T09:22:05.668 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-ceph-common-2:19.3.0-5092.g62da54b4.el9.x8 51/68 2024-09-21T09:22:05.668 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-ceph-common-2:18.2.4-871.g6a1bdf85.el9.x86 52/68 2024-09-21T09:22:05.668 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-cephfs-2:19.3.0-5092.g62da54b4.el9.x86_64 53/68 2024-09-21T09:22:05.668 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-cephfs-2:18.2.4-871.g6a1bdf85.el9.x86_64 54/68 2024-09-21T09:22:05.668 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-rados-2:19.3.0-5092.g62da54b4.el9.x86_64 55/68 2024-09-21T09:22:05.668 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-rados-2:18.2.4-871.g6a1bdf85.el9.x86_64 56/68 2024-09-21T09:22:05.669 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-rbd-2:19.3.0-5092.g62da54b4.el9.x86_64 57/68 2024-09-21T09:22:05.669 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-rbd-2:18.2.4-871.g6a1bdf85.el9.x86_64 58/68 2024-09-21T09:22:05.669 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-rgw-2:19.3.0-5092.g62da54b4.el9.x86_64 59/68 2024-09-21T09:22:05.669 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-rgw-2:18.2.4-871.g6a1bdf85.el9.x86_64 60/68 2024-09-21T09:22:05.669 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : rbd-fuse-2:19.3.0-5092.g62da54b4.el9.x86_64 61/68 2024-09-21T09:22:05.669 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : rbd-fuse-2:18.2.4-871.g6a1bdf85.el9.x86_64 62/68 2024-09-21T09:22:05.669 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : rbd-mirror-2:19.3.0-5092.g62da54b4.el9.x86_64 63/68 2024-09-21T09:22:05.669 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : rbd-mirror-2:18.2.4-871.g6a1bdf85.el9.x86_64 64/68 2024-09-21T09:22:05.669 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : rbd-nbd-2:19.3.0-5092.g62da54b4.el9.x86_64 65/68 2024-09-21T09:22:05.669 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : rbd-nbd-2:18.2.4-871.g6a1bdf85.el9.x86_64 66/68 2024-09-21T09:22:05.669 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : ceph-mgr-modules-core-2:19.3.0-5092.g62da54b4.el9. 67/68 2024-09-21T09:22:06.906 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : ceph-mgr-modules-core-2:18.2.4-871.g6a1bdf85.el9.n 68/68 2024-09-21T09:22:06.906 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:22:06.906 INFO:teuthology.orchestra.run.smithi078.stdout:Upgraded: 2024-09-21T09:22:06.906 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-2:19.3.0-5092.g62da54b4.el9.x86_64 2024-09-21T09:22:06.906 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-base-2:19.3.0-5092.g62da54b4.el9.x86_64 2024-09-21T09:22:06.906 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-common-2:19.3.0-5092.g62da54b4.el9.x86_64 2024-09-21T09:22:06.906 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-fuse-2:19.3.0-5092.g62da54b4.el9.x86_64 2024-09-21T09:22:06.906 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-immutable-object-cache-2:19.3.0-5092.g62da54b4.el9.x86_64 2024-09-21T09:22:06.906 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-mds-2:19.3.0-5092.g62da54b4.el9.x86_64 2024-09-21T09:22:06.906 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-mgr-2:19.3.0-5092.g62da54b4.el9.x86_64 2024-09-21T09:22:06.906 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-mgr-modules-core-2:19.3.0-5092.g62da54b4.el9.noarch 2024-09-21T09:22:06.907 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-mon-2:19.3.0-5092.g62da54b4.el9.x86_64 2024-09-21T09:22:06.907 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-osd-2:19.3.0-5092.g62da54b4.el9.x86_64 2024-09-21T09:22:06.907 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-radosgw-2:19.3.0-5092.g62da54b4.el9.x86_64 2024-09-21T09:22:06.907 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-selinux-2:19.3.0-5092.g62da54b4.el9.x86_64 2024-09-21T09:22:06.907 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-test-2:19.3.0-5092.g62da54b4.el9.x86_64 2024-09-21T09:22:06.907 INFO:teuthology.orchestra.run.smithi078.stdout: libcephfs-devel-2:19.3.0-5092.g62da54b4.el9.x86_64 2024-09-21T09:22:06.907 INFO:teuthology.orchestra.run.smithi078.stdout: libcephfs2-2:19.3.0-5092.g62da54b4.el9.x86_64 2024-09-21T09:22:06.907 INFO:teuthology.orchestra.run.smithi078.stdout: libcephsqlite-2:19.3.0-5092.g62da54b4.el9.x86_64 2024-09-21T09:22:06.907 INFO:teuthology.orchestra.run.smithi078.stdout: librados-devel-2:19.3.0-5092.g62da54b4.el9.x86_64 2024-09-21T09:22:06.907 INFO:teuthology.orchestra.run.smithi078.stdout: librados2-2:19.3.0-5092.g62da54b4.el9.x86_64 2024-09-21T09:22:06.907 INFO:teuthology.orchestra.run.smithi078.stdout: libradosstriper1-2:19.3.0-5092.g62da54b4.el9.x86_64 2024-09-21T09:22:06.907 INFO:teuthology.orchestra.run.smithi078.stdout: librbd1-2:19.3.0-5092.g62da54b4.el9.x86_64 2024-09-21T09:22:06.907 INFO:teuthology.orchestra.run.smithi078.stdout: librgw2-2:19.3.0-5092.g62da54b4.el9.x86_64 2024-09-21T09:22:06.907 INFO:teuthology.orchestra.run.smithi078.stdout: python3-ceph-argparse-2:19.3.0-5092.g62da54b4.el9.x86_64 2024-09-21T09:22:06.907 INFO:teuthology.orchestra.run.smithi078.stdout: python3-ceph-common-2:19.3.0-5092.g62da54b4.el9.x86_64 2024-09-21T09:22:06.908 INFO:teuthology.orchestra.run.smithi078.stdout: python3-cephfs-2:19.3.0-5092.g62da54b4.el9.x86_64 2024-09-21T09:22:06.908 INFO:teuthology.orchestra.run.smithi078.stdout: python3-rados-2:19.3.0-5092.g62da54b4.el9.x86_64 2024-09-21T09:22:06.908 INFO:teuthology.orchestra.run.smithi078.stdout: python3-rbd-2:19.3.0-5092.g62da54b4.el9.x86_64 2024-09-21T09:22:06.908 INFO:teuthology.orchestra.run.smithi078.stdout: python3-rgw-2:19.3.0-5092.g62da54b4.el9.x86_64 2024-09-21T09:22:06.908 INFO:teuthology.orchestra.run.smithi078.stdout: rbd-fuse-2:19.3.0-5092.g62da54b4.el9.x86_64 2024-09-21T09:22:06.908 INFO:teuthology.orchestra.run.smithi078.stdout: rbd-mirror-2:19.3.0-5092.g62da54b4.el9.x86_64 2024-09-21T09:22:06.908 INFO:teuthology.orchestra.run.smithi078.stdout: rbd-nbd-2:19.3.0-5092.g62da54b4.el9.x86_64 2024-09-21T09:22:06.908 INFO:teuthology.orchestra.run.smithi078.stdout:Installed: 2024-09-21T09:22:06.908 INFO:teuthology.orchestra.run.smithi078.stdout: libnbd-1.20.2-2.el9.x86_64 lua-5.4.4-4.el9.x86_64 2024-09-21T09:22:06.908 INFO:teuthology.orchestra.run.smithi078.stdout: lua-devel-5.4.4-4.el9.x86_64 luarocks-3.9.2-5.el9.noarch 2024-09-21T09:22:06.908 INFO:teuthology.orchestra.run.smithi078.stdout: python3-packaging-20.9-5.el9.noarch qatlib-24.02.0-1.el9.x86_64 2024-09-21T09:22:06.908 INFO:teuthology.orchestra.run.smithi078.stdout: qatlib-service-24.02.0-1.el9.x86_64 qatzip-libs-1.2.0-1.el9.x86_64 2024-09-21T09:22:06.908 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:22:06.908 INFO:teuthology.orchestra.run.smithi078.stdout:Complete! 2024-09-21T09:22:07.270 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:22:07.410 DEBUG:teuthology.orchestra.run.smithi078:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-09-21T09:22:07.431 INFO:teuthology.orchestra.run.smithi078.stdout:19.3.0-5092.g62da54b4.el9 2024-09-21T09:22:07.432 INFO:teuthology.packaging:The installed version of ceph is 19.3.0-5092.g62da54b4.el9 2024-09-21T09:22:07.432 INFO:teuthology.task.install:The correct ceph version 19.3.0-5092.g62da54b4 is installed. 2024-09-21T09:22:07.433 INFO:teuthology.run_tasks:Running task print... 2024-09-21T09:22:07.442 INFO:teuthology.task.print:**** done install.upgrade 2024-09-21T09:22:07.442 INFO:teuthology.run_tasks:Running task ceph.restart... 2024-09-21T09:22:07.452 DEBUG:tasks.ceph.mon.a:waiting for process to exit 2024-09-21T09:22:07.452 INFO:teuthology.orchestra.run:waiting for 300 2024-09-21T09:22:07.485 INFO:tasks.ceph.mon.a:Stopped 2024-09-21T09:22:07.485 INFO:tasks.ceph.mon.a:Restarting daemon 2024-09-21T09:22:07.485 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster ceph -i a 2024-09-21T09:22:07.528 INFO:tasks.ceph.mon.a:Started 2024-09-21T09:22:07.528 DEBUG:tasks.ceph.mon.b:waiting for process to exit 2024-09-21T09:22:07.528 INFO:teuthology.orchestra.run:waiting for 300 2024-09-21T09:22:07.540 INFO:tasks.ceph.mon.b:Stopped 2024-09-21T09:22:07.540 INFO:tasks.ceph.mon.b:Restarting daemon 2024-09-21T09:22:07.540 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster ceph -i b 2024-09-21T09:22:07.583 INFO:tasks.ceph.mon.b:Started 2024-09-21T09:22:07.583 DEBUG:tasks.ceph.mon.c:waiting for process to exit 2024-09-21T09:22:07.583 INFO:teuthology.orchestra.run:waiting for 300 2024-09-21T09:22:07.597 INFO:tasks.ceph.mon.c:Stopped 2024-09-21T09:22:07.597 INFO:tasks.ceph.mon.c:Restarting daemon 2024-09-21T09:22:07.597 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster ceph -i c 2024-09-21T09:22:07.643 INFO:tasks.ceph.mon.c:Started 2024-09-21T09:22:07.643 DEBUG:tasks.ceph.mgr.x:waiting for process to exit 2024-09-21T09:22:07.643 INFO:teuthology.orchestra.run:waiting for 300 2024-09-21T09:22:07.666 INFO:tasks.ceph.mgr.x:Stopped 2024-09-21T09:22:07.666 INFO:tasks.ceph.mgr.x:Restarting daemon 2024-09-21T09:22:07.666 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster ceph -i x 2024-09-21T09:22:07.709 INFO:tasks.ceph.mgr.x:Started 2024-09-21T09:22:07.709 DEBUG:tasks.ceph.mgr.y:waiting for process to exit 2024-09-21T09:22:07.710 INFO:teuthology.orchestra.run:waiting for 300 2024-09-21T09:22:07.732 INFO:tasks.ceph.mgr.y:Stopped 2024-09-21T09:22:07.732 INFO:tasks.ceph.mgr.y:Restarting daemon 2024-09-21T09:22:07.733 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster ceph -i y 2024-09-21T09:22:07.775 INFO:tasks.ceph.mgr.y:Started 2024-09-21T09:22:07.775 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-09-21T09:22:07.784 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-21T09:22:07.784 INFO:tasks.ceph.ceph_manager.ceph:waiting for mgr available 2024-09-21T09:22:07.784 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph mgr dump --format=json 2024-09-21T09:22:14.090 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.088+0000 7f08cb750640 1 -- 172.21.15.78:0/3240981537 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f08c4129dd0 msgr2=0x7f08b000a920 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:14.090 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.088+0000 7f08cb750640 1 --2- 172.21.15.78:0/3240981537 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f08c4129dd0 0x7f08b000a920 secure :-1 s=READY pgs=20 cs=0 l=1 rev1=1 crypto rx=0x7f08bc006b20 tx=0x7f08bc02a870 comp rx=0 tx=0).stop 2024-09-21T09:22:14.090 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.088+0000 7f08cb750640 1 -- 172.21.15.78:0/3240981537 shutdown_connections 2024-09-21T09:22:14.090 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.088+0000 7f08cb750640 1 --2- 172.21.15.78:0/3240981537 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f08c4129dd0 0x7f08b000a920 unknown :-1 s=CLOSED pgs=20 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:14.090 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.088+0000 7f08cb750640 1 --2- 172.21.15.78:0/3240981537 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f08b0003670 0x7f08b0005b00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:14.090 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.088+0000 7f08cb750640 1 --2- 172.21.15.78:0/3240981537 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f08c41294b0 0x7f08b0008350 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:14.090 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.088+0000 7f08cb750640 1 -- 172.21.15.78:0/3240981537 >> 172.21.15.78:0/3240981537 conn(0x7f08c409d480 msgr2=0x7f08c409d890 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:14.091 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.088+0000 7f08cb750640 1 -- 172.21.15.78:0/3240981537 shutdown_connections 2024-09-21T09:22:14.091 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.088+0000 7f08cb750640 1 -- 172.21.15.78:0/3240981537 wait complete. 2024-09-21T09:22:14.092 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.089+0000 7f08cb750640 1 Processor -- start 2024-09-21T09:22:14.092 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.089+0000 7f08cb750640 1 -- start start 2024-09-21T09:22:14.092 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.089+0000 7f08cb750640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f08b0003670 0x7f08c406f3e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:14.093 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.089+0000 7f08cb750640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f08c41294b0 0x7f08c406f920 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:14.093 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.089+0000 7f08cb750640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f08c4129dd0 0x7f08c406fe60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:14.093 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.089+0000 7f08cb750640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7f08c412d520 con 0x7f08b0003670 2024-09-21T09:22:14.093 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.089+0000 7f08cb750640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7f08c412d3a0 con 0x7f08c4129dd0 2024-09-21T09:22:14.093 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.090+0000 7f08cb750640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7f08c412d6a0 con 0x7f08c41294b0 2024-09-21T09:22:14.093 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.090+0000 7f08c8ceb640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f08c41294b0 0x7f08c406f920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:14.093 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.090+0000 7f08c94ec640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f08b0003670 0x7f08c406f3e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:14.093 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.090+0000 7f08c9ced640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f08c4129dd0 0x7f08c406fe60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:14.093 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.090+0000 7f08c94ec640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f08b0003670 0x7f08c406f3e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3300/0 says I am v2:172.21.15.78:50994/0 (socket says 172.21.15.78:50994) 2024-09-21T09:22:14.093 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.090+0000 7f08c94ec640 1 -- 172.21.15.78:0/2633293932 learned_addr learned my addr 172.21.15.78:0/2633293932 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:14.093 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.090+0000 7f08c8ceb640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f08c41294b0 0x7f08c406f920 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3302/0 says I am v2:172.21.15.78:53582/0 (socket says 172.21.15.78:53582) 2024-09-21T09:22:14.093 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.090+0000 7f08c94ec640 1 -- 172.21.15.78:0/2633293932 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f08c41294b0 msgr2=0x7f08c406f920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:14.094 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.090+0000 7f08c94ec640 1 --2- 172.21.15.78:0/2633293932 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f08c41294b0 0x7f08c406f920 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:14.094 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.090+0000 7f08c94ec640 1 -- 172.21.15.78:0/2633293932 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f08c4129dd0 msgr2=0x7f08c406fe60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:14.094 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.090+0000 7f08c94ec640 1 --2- 172.21.15.78:0/2633293932 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f08c4129dd0 0x7f08c406fe60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:14.094 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.090+0000 7f08c94ec640 1 -- 172.21.15.78:0/2633293932 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f08c4126360 con 0x7f08b0003670 2024-09-21T09:22:14.094 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.090+0000 7f08c9ced640 1 --2- 172.21.15.78:0/2633293932 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f08c4129dd0 0x7f08c406fe60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:22:14.094 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.090+0000 7f08c94ec640 1 --2- 172.21.15.78:0/2633293932 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f08b0003670 0x7f08c406f3e0 secure :-1 s=READY pgs=21 cs=0 l=1 rev1=1 crypto rx=0x7f08b4009fd0 tx=0x7f08b400e7c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:14.094 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.090+0000 7f08c1ffb640 1 -- 172.21.15.78:0/2633293932 <== mon.0 v2:172.21.15.78:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f08b4003b70 con 0x7f08b0003670 2024-09-21T09:22:14.094 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.090+0000 7f08c1ffb640 1 -- 172.21.15.78:0/2633293932 <== mon.0 v2:172.21.15.78:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f08b4020070 con 0x7f08b0003670 2024-09-21T09:22:14.094 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.090+0000 7f08cb750640 1 -- 172.21.15.78:0/2633293932 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f08c4070d30 con 0x7f08b0003670 2024-09-21T09:22:14.094 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.090+0000 7f08c1ffb640 1 -- 172.21.15.78:0/2633293932 <== mon.0 v2:172.21.15.78:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f08b4004020 con 0x7f08b0003670 2024-09-21T09:22:14.094 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.091+0000 7f08cb750640 1 -- 172.21.15.78:0/2633293932 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f08c40711a0 con 0x7f08b0003670 2024-09-21T09:22:14.095 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.092+0000 7f08c1ffb640 1 -- 172.21.15.78:0/2633293932 <== mon.0 v2:172.21.15.78:3300/0 4 ==== mgrmap(e 6) ==== 55351+0+0 (secure 0 0 0) 0x7f08b40041c0 con 0x7f08b0003670 2024-09-21T09:22:14.095 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.092+0000 7f08cb750640 1 -- 172.21.15.78:0/2633293932 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f08c4127a50 con 0x7f08b0003670 2024-09-21T09:22:14.095 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.092+0000 7f08c1ffb640 1 -- 172.21.15.78:0/2633293932 <== mon.0 v2:172.21.15.78:3300/0 5 ==== osd_map(19..19 src has 1..19) ==== 2812+0+0 (secure 0 0 0) 0x7f08b405f300 con 0x7f08b0003670 2024-09-21T09:22:14.095 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.092+0000 7f08c8ceb640 1 -- 172.21.15.78:0/2633293932 >> v1:172.21.15.78:6816/2602406893 conn(0x7f08b0047f80 legacy=0x7f08b004a440 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.78:6816/2602406893 2024-09-21T09:22:14.097 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.096+0000 7f08c1ffb640 1 -- 172.21.15.78:0/2633293932 <== mon.0 v2:172.21.15.78:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+148921 (secure 0 0 0) 0x7f08c4127a50 con 0x7f08b0003670 2024-09-21T09:22:14.272 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.271+0000 7f08cb750640 1 -- 172.21.15.78:0/2633293932 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "mgr dump", "format": "json"} v 0) -- 0x7f08c411a5f0 con 0x7f08b0003670 2024-09-21T09:22:14.274 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.272+0000 7f08c1ffb640 1 -- 172.21.15.78:0/2633293932 <== mon.0 v2:172.21.15.78:3300/0 7 ==== mon_command_ack([{"prefix": "mgr dump", "format": "json"}]=0 v6) ==== 74+0+107819 (secure 0 0 0) 0x7f08b4016040 con 0x7f08b0003670 2024-09-21T09:22:14.275 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:22:14.276 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.275+0000 7f08cb750640 1 -- 172.21.15.78:0/2633293932 >> v1:172.21.15.78:6816/2602406893 conn(0x7f08b0047f80 legacy=0x7f08b004a440 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-21T09:22:14.276 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.275+0000 7f08cb750640 1 -- 172.21.15.78:0/2633293932 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f08b0003670 msgr2=0x7f08c406f3e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:14.276 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.275+0000 7f08cb750640 1 --2- 172.21.15.78:0/2633293932 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f08b0003670 0x7f08c406f3e0 secure :-1 s=READY pgs=21 cs=0 l=1 rev1=1 crypto rx=0x7f08b4009fd0 tx=0x7f08b400e7c0 comp rx=0 tx=0).stop 2024-09-21T09:22:14.277 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.275+0000 7f08cb750640 1 -- 172.21.15.78:0/2633293932 shutdown_connections 2024-09-21T09:22:14.277 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.275+0000 7f08cb750640 1 --2- 172.21.15.78:0/2633293932 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f08c4129dd0 0x7f08c406fe60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:14.277 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.275+0000 7f08cb750640 1 --2- 172.21.15.78:0/2633293932 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f08c41294b0 0x7f08c406f920 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:14.277 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.275+0000 7f08cb750640 1 --2- 172.21.15.78:0/2633293932 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f08b0003670 0x7f08c406f3e0 unknown :-1 s=CLOSED pgs=21 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:14.277 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.275+0000 7f08cb750640 1 -- 172.21.15.78:0/2633293932 >> 172.21.15.78:0/2633293932 conn(0x7f08c409d480 msgr2=0x7f08c41b4d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:14.277 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.275+0000 7f08cb750640 1 -- 172.21.15.78:0/2633293932 shutdown_connections 2024-09-21T09:22:14.277 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.275+0000 7f08cb750640 1 -- 172.21.15.78:0/2633293932 wait complete. 2024-09-21T09:22:14.288 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":6,"flags":0,"active_gid":4106,"active_name":"x","active_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6816","nonce":2602406893}]},"active_addr":"172.21.15.78:6816/2602406893","active_change":"2024-09-21T09:20:41.135199+0000","active_mgr_features":4540138322906710015,"available":true,"standbys":[{"gid":4102,"name":"y","mgr_features":4540138322906710015,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["iostat","nfs","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":0,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.78:0","nonce":2453110290}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.78:0","nonce":2894754949}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.78:0","nonce":2078474014}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.78:0","nonce":1812672186}]}]} 2024-09-21T09:22:14.290 INFO:tasks.ceph.ceph_manager.ceph:mgr available! 2024-09-21T09:22:14.290 INFO:tasks.ceph.ceph_manager.ceph:waiting for all up 2024-09-21T09:22:14.290 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-09-21T09:22:14.393 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.391+0000 7f3b8583c640 1 -- 172.21.15.78:0/1091661987 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f3b80131dc0 msgr2=0x7f3b801341b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:14.393 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.391+0000 7f3b8583c640 1 --2- 172.21.15.78:0/1091661987 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f3b80131dc0 0x7f3b801341b0 secure :-1 s=READY pgs=23 cs=0 l=1 rev1=1 crypto rx=0x7f3b74005fd0 tx=0x7f3b7402a540 comp rx=0 tx=0).stop 2024-09-21T09:22:14.393 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.391+0000 7f3b8583c640 1 -- 172.21.15.78:0/1091661987 shutdown_connections 2024-09-21T09:22:14.393 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.391+0000 7f3b8583c640 1 --2- 172.21.15.78:0/1091661987 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f3b801346f0 0x7f3b80136ae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:14.394 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.391+0000 7f3b8583c640 1 --2- 172.21.15.78:0/1091661987 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f3b80131dc0 0x7f3b801341b0 unknown :-1 s=CLOSED pgs=23 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:14.394 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.391+0000 7f3b8583c640 1 -- 172.21.15.78:0/1091661987 >> 172.21.15.78:0/1091661987 conn(0x7f3b800a1800 msgr2=0x7f3b800a1c10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:14.394 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.391+0000 7f3b8583c640 1 -- 172.21.15.78:0/1091661987 shutdown_connections 2024-09-21T09:22:14.394 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.392+0000 7f3b8583c640 1 -- 172.21.15.78:0/1091661987 wait complete. 2024-09-21T09:22:14.395 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.392+0000 7f3b8583c640 1 Processor -- start 2024-09-21T09:22:14.396 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.393+0000 7f3b8583c640 1 -- start start 2024-09-21T09:22:14.396 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.393+0000 7f3b8583c640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f3b80099080 0x7f3b8009bde0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:14.396 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.393+0000 7f3b8583c640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f3b80131dc0 0x7f3b8009c320 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:14.396 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.393+0000 7f3b8583c640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f3b801346f0 0x7f3b8009c860 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:14.396 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.393+0000 7f3b8583c640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7f3b80137970 con 0x7f3b80099080 2024-09-21T09:22:14.396 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.393+0000 7f3b8583c640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7f3b801377f0 con 0x7f3b801346f0 2024-09-21T09:22:14.396 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.393+0000 7f3b8583c640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7f3b80137af0 con 0x7f3b80131dc0 2024-09-21T09:22:14.396 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.393+0000 7f3b7e7fc640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f3b80131dc0 0x7f3b8009c320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:14.396 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.393+0000 7f3b7effd640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f3b80099080 0x7f3b8009bde0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:14.397 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.393+0000 7f3b7e7fc640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f3b80131dc0 0x7f3b8009c320 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3302/0 says I am v2:172.21.15.78:53584/0 (socket says 172.21.15.78:53584) 2024-09-21T09:22:14.397 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.393+0000 7f3b7e7fc640 1 -- 172.21.15.78:0/4088797128 learned_addr learned my addr 172.21.15.78:0/4088797128 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:14.397 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.393+0000 7f3b7f7fe640 1 --2- 172.21.15.78:0/4088797128 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f3b801346f0 0x7f3b8009c860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:14.397 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.393+0000 7f3b7f7fe640 1 -- 172.21.15.78:0/4088797128 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f3b80131dc0 msgr2=0x7f3b8009c320 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:14.397 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.393+0000 7f3b7f7fe640 1 --2- 172.21.15.78:0/4088797128 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f3b80131dc0 0x7f3b8009c320 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:14.398 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.393+0000 7f3b7f7fe640 1 -- 172.21.15.78:0/4088797128 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f3b80099080 msgr2=0x7f3b8009bde0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:14.398 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.393+0000 7f3b7f7fe640 1 --2- 172.21.15.78:0/4088797128 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f3b80099080 0x7f3b8009bde0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:14.398 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.393+0000 7f3b7f7fe640 1 -- 172.21.15.78:0/4088797128 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f3b8009cda0 con 0x7f3b801346f0 2024-09-21T09:22:14.398 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.394+0000 7f3b7f7fe640 1 --2- 172.21.15.78:0/4088797128 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f3b801346f0 0x7f3b8009c860 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7f3b7800ebe0 tx=0x7f3b7800c6a0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:14.398 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.394+0000 7f3b67fff640 1 -- 172.21.15.78:0/4088797128 <== mon.1 v2:172.21.15.78:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3b78010630 con 0x7f3b801346f0 2024-09-21T09:22:14.398 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.394+0000 7f3b67fff640 1 -- 172.21.15.78:0/4088797128 <== mon.1 v2:172.21.15.78:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f3b780107d0 con 0x7f3b801346f0 2024-09-21T09:22:14.398 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.394+0000 7f3b8583c640 1 -- 172.21.15.78:0/4088797128 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f3b8009de80 con 0x7f3b801346f0 2024-09-21T09:22:14.398 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.394+0000 7f3b67fff640 1 -- 172.21.15.78:0/4088797128 <== mon.1 v2:172.21.15.78:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3b78010c00 con 0x7f3b801346f0 2024-09-21T09:22:14.398 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.394+0000 7f3b8583c640 1 -- 172.21.15.78:0/4088797128 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7f3b8009d480 con 0x7f3b801346f0 2024-09-21T09:22:14.399 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.395+0000 7f3b8583c640 1 -- 172.21.15.78:0/4088797128 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f3b80099460 con 0x7f3b801346f0 2024-09-21T09:22:14.399 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.395+0000 7f3b67fff640 1 -- 172.21.15.78:0/4088797128 <== mon.1 v2:172.21.15.78:3301/0 4 ==== mgrmap(e 6) ==== 55351+0+0 (secure 0 0 0) 0x7f3b78021430 con 0x7f3b801346f0 2024-09-21T09:22:14.399 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.395+0000 7f3b67fff640 1 -- 172.21.15.78:0/4088797128 <== mon.1 v2:172.21.15.78:3301/0 5 ==== osd_map(19..19 src has 1..19) ==== 2812+0+0 (secure 0 0 0) 0x7f3b7805daf0 con 0x7f3b801346f0 2024-09-21T09:22:14.399 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.395+0000 7f3b7effd640 1 -- 172.21.15.78:0/4088797128 >> v1:172.21.15.78:6816/2602406893 conn(0x7f3b6c0475f0 legacy=0x7f3b6c049ab0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.78:6816/2602406893 2024-09-21T09:22:14.400 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.399+0000 7f3b67fff640 1 -- 172.21.15.78:0/4088797128 <== mon.1 v2:172.21.15.78:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+148921 (secure 0 0 0) 0x7f3b7805cc80 con 0x7f3b801346f0 2024-09-21T09:22:14.568 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.567+0000 7f3b8583c640 1 -- 172.21.15.78:0/4088797128 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7f3b8009e140 con 0x7f3b801346f0 2024-09-21T09:22:14.569 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.567+0000 7f3b67fff640 1 -- 172.21.15.78:0/4088797128 <== mon.1 v2:172.21.15.78:3301/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v19) ==== 74+0+8823 (secure 0 0 0) 0x7f3b7803d260 con 0x7f3b801346f0 2024-09-21T09:22:14.569 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:22:14.569 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":19,"fsid":"3633ad52-1063-4406-97d9-d53c339f43ca","created":"2024-09-21T09:20:38.248415+0000","modified":"2024-09-21T09:21:04.497780+0000","last_up_change":"2024-09-21T09:20:49.276020+0000","last_in_change":"2024-09-21T09:20:40.330925+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":4,"require_min_compat_client":"reef","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T09:20:51.169034+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-21T09:20:54.918007+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":17,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"6b18d222-0705-4495-a01c-3e2ba2ab4cff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6808","nonce":1449157914}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6809","nonce":1449157914}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6811","nonce":1449157914}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6810","nonce":1449157914}]},"public_addr":"172.21.15.78:6808/1449157914","cluster_addr":"172.21.15.78:6809/1449157914","heartbeat_back_addr":"172.21.15.78:6811/1449157914","heartbeat_front_addr":"172.21.15.78:6810/1449157914","state":["exists","up"]},{"osd":1,"uuid":"2d15b0fa-66c2-4bda-b64b-d72cbbb16fb1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6800","nonce":3190796132}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6801","nonce":3190796132}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6805","nonce":3190796132}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6803","nonce":3190796132}]},"public_addr":"172.21.15.78:6800/3190796132","cluster_addr":"172.21.15.78:6801/3190796132","heartbeat_back_addr":"172.21.15.78:6805/3190796132","heartbeat_front_addr":"172.21.15.78:6803/3190796132","state":["exists","up"]},{"osd":2,"uuid":"0bf75d8f-017d-488d-8a2e-c3fa3b2a29c2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6812","nonce":3442117044}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6813","nonce":3442117044}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6815","nonce":3442117044}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6814","nonce":3442117044}]},"public_addr":"172.21.15.78:6812/3442117044","cluster_addr":"172.21.15.78:6813/3442117044","heartbeat_back_addr":"172.21.15.78:6815/3442117044","heartbeat_front_addr":"172.21.15.78:6814/3442117044","state":["exists","up"]},{"osd":3,"uuid":"5e3788c8-b89a-4614-b3ec-7ee1250ddbe0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6802","nonce":4276488585}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6804","nonce":4276488585}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6807","nonce":4276488585}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6806","nonce":4276488585}]},"public_addr":"172.21.15.78:6802/4276488585","cluster_addr":"172.21.15.78:6804/4276488585","heartbeat_back_addr":"172.21.15.78:6807/4276488585","heartbeat_front_addr":"172.21.15.78:6806/4276488585","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:20:45.499319+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:20:45.489695+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:20:45.541959+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:20:45.476915+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T09:22:14.571 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.569+0000 7f3b8583c640 1 -- 172.21.15.78:0/4088797128 >> v1:172.21.15.78:6816/2602406893 conn(0x7f3b6c0475f0 legacy=0x7f3b6c049ab0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-21T09:22:14.571 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.569+0000 7f3b8583c640 1 -- 172.21.15.78:0/4088797128 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f3b801346f0 msgr2=0x7f3b8009c860 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:14.571 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.569+0000 7f3b8583c640 1 --2- 172.21.15.78:0/4088797128 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f3b801346f0 0x7f3b8009c860 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7f3b7800ebe0 tx=0x7f3b7800c6a0 comp rx=0 tx=0).stop 2024-09-21T09:22:14.571 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.569+0000 7f3b8583c640 1 -- 172.21.15.78:0/4088797128 shutdown_connections 2024-09-21T09:22:14.571 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.569+0000 7f3b8583c640 1 --2- 172.21.15.78:0/4088797128 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f3b801346f0 0x7f3b8009c860 unknown :-1 s=CLOSED pgs=16 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:14.571 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.569+0000 7f3b8583c640 1 --2- 172.21.15.78:0/4088797128 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f3b80131dc0 0x7f3b8009c320 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:14.571 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.569+0000 7f3b8583c640 1 --2- 172.21.15.78:0/4088797128 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f3b80099080 0x7f3b8009bde0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:14.572 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.569+0000 7f3b8583c640 1 -- 172.21.15.78:0/4088797128 >> 172.21.15.78:0/4088797128 conn(0x7f3b800a1800 msgr2=0x7f3b80135870 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:14.572 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.570+0000 7f3b8583c640 1 -- 172.21.15.78:0/4088797128 shutdown_connections 2024-09-21T09:22:14.572 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.570+0000 7f3b8583c640 1 -- 172.21.15.78:0/4088797128 wait complete. 2024-09-21T09:22:14.582 INFO:tasks.ceph.ceph_manager.ceph:all up! 2024-09-21T09:22:14.583 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-09-21T09:22:14.690 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.688+0000 7faf5757f640 1 -- 172.21.15.78:0/3614546255 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7faf50099080 msgr2=0x7faf50099460 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:14.690 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.688+0000 7faf5757f640 1 --2- 172.21.15.78:0/3614546255 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7faf50099080 0x7faf50099460 secure :-1 s=READY pgs=13 cs=0 l=1 rev1=1 crypto rx=0x7faf4c009830 tx=0x7faf4c02a480 comp rx=0 tx=0).stop 2024-09-21T09:22:14.690 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.688+0000 7faf5757f640 1 -- 172.21.15.78:0/3614546255 shutdown_connections 2024-09-21T09:22:14.690 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.688+0000 7faf5757f640 1 --2- 172.21.15.78:0/3614546255 >> v2:172.21.15.78:3300/0 conn(0x7faf500999a0 0x7faf5012eab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:14.690 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.688+0000 7faf5757f640 1 --2- 172.21.15.78:0/3614546255 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7faf50099080 0x7faf50099460 unknown :-1 s=CLOSED pgs=13 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:14.690 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.688+0000 7faf5757f640 1 -- 172.21.15.78:0/3614546255 >> 172.21.15.78:0/3614546255 conn(0x7faf500a1800 msgr2=0x7faf500a1c10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:14.690 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.688+0000 7faf5757f640 1 -- 172.21.15.78:0/3614546255 shutdown_connections 2024-09-21T09:22:14.690 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.688+0000 7faf5757f640 1 -- 172.21.15.78:0/3614546255 wait complete. 2024-09-21T09:22:14.691 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.689+0000 7faf5757f640 1 Processor -- start 2024-09-21T09:22:14.691 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.689+0000 7faf5757f640 1 -- start start 2024-09-21T09:22:14.692 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.689+0000 7faf5757f640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7faf50099080 0x7faf50071320 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:14.692 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.689+0000 7faf5757f640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7faf500999a0 0x7faf50071860 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:14.692 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.689+0000 7faf5757f640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7faf5012eff0 0x7faf50071da0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:14.692 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.689+0000 7faf5757f640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7faf50137610 con 0x7faf50099080 2024-09-21T09:22:14.692 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.689+0000 7faf5757f640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7faf50137490 con 0x7faf500999a0 2024-09-21T09:22:14.692 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.689+0000 7faf5757f640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7faf50137790 con 0x7faf5012eff0 2024-09-21T09:22:14.693 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.689+0000 7faf55b1c640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7faf5012eff0 0x7faf50071da0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:14.693 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.689+0000 7faf54b1a640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7faf500999a0 0x7faf50071860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:14.693 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.689+0000 7faf54b1a640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7faf500999a0 0x7faf50071860 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3301/0 says I am v2:172.21.15.78:60558/0 (socket says 172.21.15.78:60558) 2024-09-21T09:22:14.693 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.689+0000 7faf54b1a640 1 -- 172.21.15.78:0/711482315 learned_addr learned my addr 172.21.15.78:0/711482315 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:14.693 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.689+0000 7faf5531b640 1 --2- 172.21.15.78:0/711482315 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7faf50099080 0x7faf50071320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:14.693 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.690+0000 7faf54b1a640 1 -- 172.21.15.78:0/711482315 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7faf5012eff0 msgr2=0x7faf50071da0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:14.693 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.690+0000 7faf54b1a640 1 --2- 172.21.15.78:0/711482315 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7faf5012eff0 0x7faf50071da0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:14.693 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.690+0000 7faf54b1a640 1 -- 172.21.15.78:0/711482315 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7faf50099080 msgr2=0x7faf50071320 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:14.693 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.690+0000 7faf54b1a640 1 --2- 172.21.15.78:0/711482315 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7faf50099080 0x7faf50071320 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:14.693 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.690+0000 7faf54b1a640 1 -- 172.21.15.78:0/711482315 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7faf50135f30 con 0x7faf500999a0 2024-09-21T09:22:14.693 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.690+0000 7faf55b1c640 1 --2- 172.21.15.78:0/711482315 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7faf5012eff0 0x7faf50071da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:22:14.693 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.690+0000 7faf5531b640 1 --2- 172.21.15.78:0/711482315 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7faf50099080 0x7faf50071320 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:22:14.693 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.690+0000 7faf54b1a640 1 --2- 172.21.15.78:0/711482315 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7faf500999a0 0x7faf50071860 secure :-1 s=READY pgs=17 cs=0 l=1 rev1=1 crypto rx=0x7faf4000e9f0 tx=0x7faf4000eeb0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:14.693 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.690+0000 7faf45ffb640 1 -- 172.21.15.78:0/711482315 <== mon.1 v2:172.21.15.78:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7faf4000cde0 con 0x7faf500999a0 2024-09-21T09:22:14.694 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.690+0000 7faf5757f640 1 -- 172.21.15.78:0/711482315 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7faf500732e0 con 0x7faf500999a0 2024-09-21T09:22:14.694 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.690+0000 7faf45ffb640 1 -- 172.21.15.78:0/711482315 <== mon.1 v2:172.21.15.78:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7faf40004020 con 0x7faf500999a0 2024-09-21T09:22:14.694 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.690+0000 7faf45ffb640 1 -- 172.21.15.78:0/711482315 <== mon.1 v2:172.21.15.78:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7faf40004450 con 0x7faf500999a0 2024-09-21T09:22:14.694 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.690+0000 7faf5757f640 1 -- 172.21.15.78:0/711482315 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7faf50073d20 con 0x7faf500999a0 2024-09-21T09:22:14.694 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.692+0000 7faf5757f640 1 -- 172.21.15.78:0/711482315 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7faf50128d40 con 0x7faf500999a0 2024-09-21T09:22:14.695 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.692+0000 7faf45ffb640 1 -- 172.21.15.78:0/711482315 <== mon.1 v2:172.21.15.78:3301/0 4 ==== mgrmap(e 6) ==== 55351+0+0 (secure 0 0 0) 0x7faf400035f0 con 0x7faf500999a0 2024-09-21T09:22:14.695 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.692+0000 7faf45ffb640 1 -- 172.21.15.78:0/711482315 <== mon.1 v2:172.21.15.78:3301/0 5 ==== osd_map(19..19 src has 1..19) ==== 2812+0+0 (secure 0 0 0) 0x7faf4005d630 con 0x7faf500999a0 2024-09-21T09:22:14.695 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.692+0000 7faf5531b640 1 -- 172.21.15.78:0/711482315 >> v1:172.21.15.78:6816/2602406893 conn(0x7faf340475f0 legacy=0x7faf34049ab0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.78:6816/2602406893 2024-09-21T09:22:14.700 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.698+0000 7faf45ffb640 1 -- 172.21.15.78:0/711482315 <== mon.1 v2:172.21.15.78:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+148921 (secure 0 0 0) 0x7faf50128d40 con 0x7faf500999a0 2024-09-21T09:22:14.845 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.843+0000 7faf5757f640 1 -- 172.21.15.78:0/711482315 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7faf50099460 con 0x7faf500999a0 2024-09-21T09:22:14.845 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.844+0000 7faf45ffb640 1 -- 172.21.15.78:0/711482315 <== mon.1 v2:172.21.15.78:3301/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v19) ==== 74+0+8823 (secure 0 0 0) 0x7faf4005c7c0 con 0x7faf500999a0 2024-09-21T09:22:14.846 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:22:14.846 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":19,"fsid":"3633ad52-1063-4406-97d9-d53c339f43ca","created":"2024-09-21T09:20:38.248415+0000","modified":"2024-09-21T09:21:04.497780+0000","last_up_change":"2024-09-21T09:20:49.276020+0000","last_in_change":"2024-09-21T09:20:40.330925+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":4,"require_min_compat_client":"reef","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T09:20:51.169034+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-21T09:20:54.918007+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":17,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"6b18d222-0705-4495-a01c-3e2ba2ab4cff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6808","nonce":1449157914}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6809","nonce":1449157914}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6811","nonce":1449157914}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6810","nonce":1449157914}]},"public_addr":"172.21.15.78:6808/1449157914","cluster_addr":"172.21.15.78:6809/1449157914","heartbeat_back_addr":"172.21.15.78:6811/1449157914","heartbeat_front_addr":"172.21.15.78:6810/1449157914","state":["exists","up"]},{"osd":1,"uuid":"2d15b0fa-66c2-4bda-b64b-d72cbbb16fb1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6800","nonce":3190796132}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6801","nonce":3190796132}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6805","nonce":3190796132}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6803","nonce":3190796132}]},"public_addr":"172.21.15.78:6800/3190796132","cluster_addr":"172.21.15.78:6801/3190796132","heartbeat_back_addr":"172.21.15.78:6805/3190796132","heartbeat_front_addr":"172.21.15.78:6803/3190796132","state":["exists","up"]},{"osd":2,"uuid":"0bf75d8f-017d-488d-8a2e-c3fa3b2a29c2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6812","nonce":3442117044}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6813","nonce":3442117044}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6815","nonce":3442117044}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6814","nonce":3442117044}]},"public_addr":"172.21.15.78:6812/3442117044","cluster_addr":"172.21.15.78:6813/3442117044","heartbeat_back_addr":"172.21.15.78:6815/3442117044","heartbeat_front_addr":"172.21.15.78:6814/3442117044","state":["exists","up"]},{"osd":3,"uuid":"5e3788c8-b89a-4614-b3ec-7ee1250ddbe0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6802","nonce":4276488585}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6804","nonce":4276488585}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6807","nonce":4276488585}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6806","nonce":4276488585}]},"public_addr":"172.21.15.78:6802/4276488585","cluster_addr":"172.21.15.78:6804/4276488585","heartbeat_back_addr":"172.21.15.78:6807/4276488585","heartbeat_front_addr":"172.21.15.78:6806/4276488585","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:20:45.499319+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:20:45.489695+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:20:45.541959+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:20:45.476915+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T09:22:14.848 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.846+0000 7faf5757f640 1 -- 172.21.15.78:0/711482315 >> v1:172.21.15.78:6816/2602406893 conn(0x7faf340475f0 legacy=0x7faf34049ab0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-21T09:22:14.848 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.846+0000 7faf5757f640 1 -- 172.21.15.78:0/711482315 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7faf500999a0 msgr2=0x7faf50071860 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:14.848 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.846+0000 7faf5757f640 1 --2- 172.21.15.78:0/711482315 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7faf500999a0 0x7faf50071860 secure :-1 s=READY pgs=17 cs=0 l=1 rev1=1 crypto rx=0x7faf4000e9f0 tx=0x7faf4000eeb0 comp rx=0 tx=0).stop 2024-09-21T09:22:14.848 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.846+0000 7faf5757f640 1 -- 172.21.15.78:0/711482315 shutdown_connections 2024-09-21T09:22:14.848 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.846+0000 7faf5757f640 1 --2- 172.21.15.78:0/711482315 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7faf5012eff0 0x7faf50071da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:14.848 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.846+0000 7faf5757f640 1 --2- 172.21.15.78:0/711482315 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7faf500999a0 0x7faf50071860 unknown :-1 s=CLOSED pgs=17 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:14.848 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.846+0000 7faf5757f640 1 --2- 172.21.15.78:0/711482315 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7faf50099080 0x7faf50071320 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:14.848 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.846+0000 7faf5757f640 1 -- 172.21.15.78:0/711482315 >> 172.21.15.78:0/711482315 conn(0x7faf500a1800 msgr2=0x7faf5012e2d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:14.848 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.846+0000 7faf5757f640 1 -- 172.21.15.78:0/711482315 shutdown_connections 2024-09-21T09:22:14.848 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:14.847+0000 7faf5757f640 1 -- 172.21.15.78:0/711482315 wait complete. 2024-09-21T09:22:14.864 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.0 flush_pg_stats 2024-09-21T09:22:14.864 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.1 flush_pg_stats 2024-09-21T09:22:14.865 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.2 flush_pg_stats 2024-09-21T09:22:14.865 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.3 flush_pg_stats 2024-09-21T09:22:15.029 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.026+0000 7fba45ba1640 1 -- 172.21.15.78:0/3181193636 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fba400973f0 msgr2=0x7fba4012be30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:15.029 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.026+0000 7fba45ba1640 1 --2- 172.21.15.78:0/3181193636 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fba400973f0 0x7fba4012be30 secure :-1 s=READY pgs=27 cs=0 l=1 rev1=1 crypto rx=0x7fba38005fd0 tx=0x7fba3802a810 comp rx=0 tx=0).stop 2024-09-21T09:22:15.029 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.027+0000 7fba45ba1640 1 -- 172.21.15.78:0/3181193636 shutdown_connections 2024-09-21T09:22:15.029 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.027+0000 7fba45ba1640 1 --2- 172.21.15.78:0/3181193636 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fba400973f0 0x7fba4012be30 unknown :-1 s=CLOSED pgs=27 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:15.029 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.027+0000 7fba45ba1640 1 --2- 172.21.15.78:0/3181193636 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fba40095dd0 0x7fba400961b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:15.030 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.027+0000 7fba45ba1640 1 -- 172.21.15.78:0/3181193636 >> 172.21.15.78:0/3181193636 conn(0x7fba4009c470 msgr2=0x7fba4009c880 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:15.030 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.027+0000 7fba45ba1640 1 -- 172.21.15.78:0/3181193636 shutdown_connections 2024-09-21T09:22:15.030 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.027+0000 7fba45ba1640 1 -- 172.21.15.78:0/3181193636 wait complete. 2024-09-21T09:22:15.031 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.028+0000 7fba45ba1640 1 Processor -- start 2024-09-21T09:22:15.031 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.028+0000 7fba45ba1640 1 -- start start 2024-09-21T09:22:15.031 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.029+0000 7fba45ba1640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fba40095dd0 0x7fba4012fba0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:15.031 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.029+0000 7fba45ba1640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fba400966f0 0x7fba401300e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:15.031 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.029+0000 7fba45ba1640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fba400973f0 0x7fba40130620 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:15.032 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.029+0000 7fba45ba1640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7fba4006ee60 con 0x7fba40095dd0 2024-09-21T09:22:15.032 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.029+0000 7fba45ba1640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7fba4006ece0 con 0x7fba400973f0 2024-09-21T09:22:15.032 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.029+0000 7fba45ba1640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7fba4006efe0 con 0x7fba400966f0 2024-09-21T09:22:15.032 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.029+0000 7f086e2f9640 1 -- 172.21.15.78:0/1884100667 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f0850007420 msgr2=0x7f085000dbd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:15.032 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.029+0000 7f086e2f9640 1 --2- 172.21.15.78:0/1884100667 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f0850007420 0x7f085000dbd0 secure :-1 s=READY pgs=18 cs=0 l=1 rev1=1 crypto rx=0x7f086000c640 tx=0x7f0850007a40 comp rx=0 tx=0).stop 2024-09-21T09:22:15.033 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.029+0000 7fba3ffff640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fba400973f0 0x7fba40130620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:15.033 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.029+0000 7fba3ffff640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fba400973f0 0x7fba40130620 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3301/0 says I am v2:172.21.15.78:60572/0 (socket says 172.21.15.78:60572) 2024-09-21T09:22:15.033 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.029+0000 7fba3ffff640 1 -- 172.21.15.78:0/2207083386 learned_addr learned my addr 172.21.15.78:0/2207083386 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:15.033 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.029+0000 7fba3f7fe640 1 --2- 172.21.15.78:0/2207083386 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fba40095dd0 0x7fba4012fba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:15.033 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.029+0000 7fba37fff640 1 --2- 172.21.15.78:0/2207083386 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fba400966f0 0x7fba401300e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:15.033 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.029+0000 7fba3ffff640 1 -- 172.21.15.78:0/2207083386 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fba400966f0 msgr2=0x7fba401300e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:15.033 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.029+0000 7fba3ffff640 1 --2- 172.21.15.78:0/2207083386 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fba400966f0 0x7fba401300e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:15.033 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.029+0000 7fba3ffff640 1 -- 172.21.15.78:0/2207083386 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fba40095dd0 msgr2=0x7fba4012fba0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:15.034 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.029+0000 7fba3ffff640 1 --2- 172.21.15.78:0/2207083386 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fba40095dd0 0x7fba4012fba0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:15.034 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.029+0000 7fba3ffff640 1 -- 172.21.15.78:0/2207083386 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fba4012e530 con 0x7fba400973f0 2024-09-21T09:22:15.034 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.029+0000 7f086c896640 1 -- 172.21.15.78:0/1884100667 reap_dead start 2024-09-21T09:22:15.034 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.030+0000 7f086e2f9640 1 -- 172.21.15.78:0/1884100667 shutdown_connections 2024-09-21T09:22:15.034 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.030+0000 7f086e2f9640 1 -- 172.21.15.78:0/1884100667 >> 172.21.15.78:0/1884100667 conn(0x7f086809dde0 msgr2=0x7f086809e1f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:15.034 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.030+0000 7f086e2f9640 1 -- 172.21.15.78:0/1884100667 shutdown_connections 2024-09-21T09:22:15.035 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.030+0000 7fba3ffff640 1 --2- 172.21.15.78:0/2207083386 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fba400973f0 0x7fba40130620 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7fba38009b30 tx=0x7fba380315b0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:15.035 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.030+0000 7f086e2f9640 1 -- 172.21.15.78:0/1884100667 wait complete. 2024-09-21T09:22:15.036 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.030+0000 7fba3cff9640 1 -- 172.21.15.78:0/2207083386 <== mon.1 v2:172.21.15.78:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fba38041070 con 0x7fba400973f0 2024-09-21T09:22:15.036 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.030+0000 7fba45ba1640 1 -- 172.21.15.78:0/2207083386 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fba40130b60 con 0x7fba400973f0 2024-09-21T09:22:15.036 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.030+0000 7fba45ba1640 1 -- 172.21.15.78:0/2207083386 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7fba4012ea20 con 0x7fba400973f0 2024-09-21T09:22:15.036 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.030+0000 7fba3cff9640 1 -- 172.21.15.78:0/2207083386 <== mon.1 v2:172.21.15.78:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fba38031aa0 con 0x7fba400973f0 2024-09-21T09:22:15.036 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.030+0000 7fba3cff9640 1 -- 172.21.15.78:0/2207083386 <== mon.1 v2:172.21.15.78:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fba38037040 con 0x7fba400973f0 2024-09-21T09:22:15.036 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.031+0000 7fba35ffb640 1 -- 172.21.15.78:0/2207083386 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_get_version(what=osdmap handle=1) -- 0x7fba4012bc30 con 0x7fba400973f0 2024-09-21T09:22:15.037 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.031+0000 7f086e2f9640 1 Processor -- start 2024-09-21T09:22:15.037 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.032+0000 7f086e2f9640 1 -- start start 2024-09-21T09:22:15.037 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.033+0000 7f086e2f9640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f0850007420 0x7f086808bf80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:15.037 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.033+0000 7f0867fff640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f0850007420 0x7f086808bf80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:15.038 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.033+0000 7f0867fff640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f0850007420 0x7f086808bf80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3302/0 says I am v2:172.21.15.78:53652/0 (socket says 172.21.15.78:53652) 2024-09-21T09:22:15.038 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.033+0000 7f0867fff640 1 -- 172.21.15.78:0/434505038 learned_addr learned my addr 172.21.15.78:0/434505038 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:15.038 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.033+0000 7fba3cff9640 1 -- 172.21.15.78:0/2207083386 <== mon.1 v2:172.21.15.78:3301/0 4 ==== mgrmap(e 6) ==== 55351+0+0 (secure 0 0 0) 0x7fba38044070 con 0x7fba400973f0 2024-09-21T09:22:15.038 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.033+0000 7fba3cff9640 1 -- 172.21.15.78:0/2207083386 <== mon.1 v2:172.21.15.78:3301/0 5 ==== osd_map(19..19 src has 1..19) ==== 2812+0+0 (secure 0 0 0) 0x7fba3807d8f0 con 0x7fba400973f0 2024-09-21T09:22:15.038 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.033+0000 7fba3cff9640 1 -- 172.21.15.78:0/2207083386 --> v1:172.21.15.78:6808/1449157914 -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7fba38039e50 con 0x7fba2004dc70 2024-09-21T09:22:15.039 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.033+0000 7fba3f7fe640 1 -- 172.21.15.78:0/2207083386 >> v1:172.21.15.78:6816/2602406893 conn(0x7fba200475f0 legacy=0x7fba20049ab0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.78:6816/2602406893 2024-09-21T09:22:15.039 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.034+0000 7fba3cff9640 1 -- 172.21.15.78:0/2207083386 <== mon.1 v2:172.21.15.78:3301/0 6 ==== mon_get_version_reply(handle=1 version=19) ==== 24+0+0 (secure 0 0 0) 0x7fba38048480 con 0x7fba400973f0 2024-09-21T09:22:15.039 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.034+0000 7f086e2f9640 1 --2- 172.21.15.78:0/434505038 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f086808c4c0 0x7f086808c900 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:15.039 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.034+0000 7f08677fe640 1 --2- 172.21.15.78:0/434505038 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f086808c4c0 0x7f086808c900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:15.039 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.034+0000 7f086e2f9640 1 --2- 172.21.15.78:0/434505038 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f08681c4680 0x7f08681c6a70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:15.040 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.035+0000 7fb26095f640 1 -- 172.21.15.78:0/4231621876 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fb25c12f2f0 msgr2=0x7fb25c12f6d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:15.040 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.035+0000 7fb26095f640 1 --2- 172.21.15.78:0/4231621876 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fb25c12f2f0 0x7fb25c12f6d0 secure :-1 s=READY pgs=28 cs=0 l=1 rev1=1 crypto rx=0x7fb254005fd0 tx=0x7fb25402a980 comp rx=0 tx=0).stop 2024-09-21T09:22:15.040 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.035+0000 7fba3cff9640 1 -- 172.21.15.78:0/2207083386 <== osd.0 v1:172.21.15.78:6808/1449157914 1 ==== command_reply(tid 1: 0 ) ==== 8+0+26118 (unknown 0 0 1686565167) 0x7fba38039e50 con 0x7fba2004dc70 2024-09-21T09:22:15.040 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.035+0000 7fb26095f640 1 -- 172.21.15.78:0/4231621876 shutdown_connections 2024-09-21T09:22:15.040 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.035+0000 7f086c896640 1 --2- 172.21.15.78:0/434505038 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f08681c4680 0x7f08681c6a70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:15.041 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.035+0000 7fb26095f640 1 --2- 172.21.15.78:0/4231621876 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fb25c12f2f0 0x7fb25c12f6d0 unknown :-1 s=CLOSED pgs=28 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:15.041 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.035+0000 7fb26095f640 1 --2- 172.21.15.78:0/4231621876 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fb25c073980 0x7fb25c12edb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:15.041 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.035+0000 7fb26095f640 1 -- 172.21.15.78:0/4231621876 >> 172.21.15.78:0/4231621876 conn(0x7fb25c09e490 msgr2=0x7fb25c09e8a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:15.042 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.035+0000 7f086e2f9640 1 -- 172.21.15.78:0/434505038 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7f0868076620 con 0x7f086808c4c0 2024-09-21T09:22:15.042 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.035+0000 7f086e2f9640 1 -- 172.21.15.78:0/434505038 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7f08680764a0 con 0x7f08681c4680 2024-09-21T09:22:15.042 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.035+0000 7f086e2f9640 1 -- 172.21.15.78:0/434505038 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7f08680767a0 con 0x7f0850007420 2024-09-21T09:22:15.042 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.036+0000 7f08677fe640 1 -- 172.21.15.78:0/434505038 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f0850007420 msgr2=0x7f086808bf80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:15.042 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.036+0000 7f08677fe640 1 --2- 172.21.15.78:0/434505038 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f0850007420 0x7f086808bf80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:15.042 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.036+0000 7f08677fe640 1 -- 172.21.15.78:0/434505038 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f08681c4680 msgr2=0x7f08681c6a70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:15.042 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.036+0000 7f08677fe640 1 --2- 172.21.15.78:0/434505038 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f08681c4680 0x7f08681c6a70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:15.042 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.036+0000 7f08677fe640 1 -- 172.21.15.78:0/434505038 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f08681280c0 con 0x7f086808c4c0 2024-09-21T09:22:15.042 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.036+0000 7f0867fff640 1 --2- 172.21.15.78:0/434505038 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f0850007420 0x7f086808bf80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-09-21T09:22:15.042 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.036+0000 7f086c896640 1 --2- 172.21.15.78:0/434505038 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f08681c4680 0x7f08681c6a70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-09-21T09:22:15.043 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.036+0000 7fb26095f640 1 -- 172.21.15.78:0/4231621876 shutdown_connections 2024-09-21T09:22:15.043 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.037+0000 7f08677fe640 1 --2- 172.21.15.78:0/434505038 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f086808c4c0 0x7f086808c900 secure :-1 s=READY pgs=29 cs=0 l=1 rev1=1 crypto rx=0x7f08680704e0 tx=0x7f085c00b730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:15.043 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.037+0000 7fb26095f640 1 -- 172.21.15.78:0/4231621876 wait complete. 2024-09-21T09:22:15.043 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.037+0000 7f0864ff9640 1 -- 172.21.15.78:0/434505038 <== mon.0 v2:172.21.15.78:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f085c01c070 con 0x7f086808c4c0 2024-09-21T09:22:15.044 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.037+0000 7f0864ff9640 1 -- 172.21.15.78:0/434505038 <== mon.0 v2:172.21.15.78:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f085c0174e0 con 0x7f086808c4c0 2024-09-21T09:22:15.044 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.037+0000 7f0864ff9640 1 -- 172.21.15.78:0/434505038 <== mon.0 v2:172.21.15.78:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f085c0178b0 con 0x7f086808c4c0 2024-09-21T09:22:15.044 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.039+0000 7fb26095f640 1 Processor -- start 2024-09-21T09:22:15.044 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.039+0000 7fb26095f640 1 -- start start 2024-09-21T09:22:15.044 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.039+0000 7fb26095f640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fb25c073980 0x7fb25c091240 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:15.044 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.039+0000 7fb26095f640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fb25c12b640 0x7fb25c091780 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:15.044 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.039+0000 7fb26095f640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fb25c12f2f0 0x7fb25c0961e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:15.044 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.039+0000 7fb26095f640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7fb25c13be70 con 0x7fb25c12f2f0 2024-09-21T09:22:15.044 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.039+0000 7fb26095f640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7fb25c13bcf0 con 0x7fb25c073980 2024-09-21T09:22:15.044 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.039+0000 7fb26095f640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7fb25c13bff0 con 0x7fb25c12b640 2024-09-21T09:22:15.045 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.039+0000 7f086e2f9640 1 -- 172.21.15.78:0/434505038 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f0868127b30 con 0x7f086808c4c0 2024-09-21T09:22:15.045 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.039+0000 7f086e2f9640 1 -- 172.21.15.78:0/434505038 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f086808d400 con 0x7f086808c4c0 2024-09-21T09:22:15.046 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.040+0000 7fb25a59c640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fb25c073980 0x7fb25c091240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:15.046 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.040+0000 7fb25a59c640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fb25c073980 0x7fb25c091240 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3301/0 says I am v2:172.21.15.78:60606/0 (socket says 172.21.15.78:60606) 2024-09-21T09:22:15.046 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.040+0000 7fb25a59c640 1 -- 172.21.15.78:0/4247053430 learned_addr learned my addr 172.21.15.78:0/4247053430 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:15.046 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.040+0000 7fb25ad9d640 1 --2- 172.21.15.78:0/4247053430 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fb25c12f2f0 0x7fb25c0961e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:15.046 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.041+0000 7fb25ad9d640 1 -- 172.21.15.78:0/4247053430 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fb25c12b640 msgr2=0x7fb25c091780 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-21T09:22:15.046 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.041+0000 7fb25ad9d640 1 --2- 172.21.15.78:0/4247053430 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fb25c12b640 0x7fb25c091780 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:15.046 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.041+0000 7fb25ad9d640 1 -- 172.21.15.78:0/4247053430 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fb25c073980 msgr2=0x7fb25c091240 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:15.046 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.041+0000 7fb25ad9d640 1 --2- 172.21.15.78:0/4247053430 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fb25c073980 0x7fb25c091240 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:15.046 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.041+0000 7fb25ad9d640 1 -- 172.21.15.78:0/4247053430 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fb25c097a50 con 0x7fb25c12f2f0 2024-09-21T09:22:15.046 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.042+0000 7fb25ad9d640 1 --2- 172.21.15.78:0/4247053430 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fb25c12f2f0 0x7fb25c0961e0 secure :-1 s=READY pgs=30 cs=0 l=1 rev1=1 crypto rx=0x7fb25000b3e0 tx=0x7fb25000b8a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:15.046 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.042+0000 7fb24affd640 1 -- 172.21.15.78:0/4247053430 <== mon.0 v2:172.21.15.78:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb250015020 con 0x7fb25c12f2f0 2024-09-21T09:22:15.047 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.042+0000 7f0864ff9640 1 -- 172.21.15.78:0/434505038 <== mon.0 v2:172.21.15.78:3300/0 4 ==== mgrmap(e 6) ==== 55351+0+0 (secure 0 0 0) 0x7f085c01f070 con 0x7f086808c4c0 2024-09-21T09:22:15.047 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.043+0000 7f0864ff9640 1 -- 172.21.15.78:0/434505038 <== mon.0 v2:172.21.15.78:3300/0 5 ==== osd_map(19..19 src has 1..19) ==== 2812+0+0 (secure 0 0 0) 0x7f085c05d290 con 0x7f086808c4c0 2024-09-21T09:22:15.047 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.043+0000 7f0867fff640 1 -- 172.21.15.78:0/434505038 >> v1:172.21.15.78:6816/2602406893 conn(0x7f08540476f0 legacy=0x7f0854049bb0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.78:6816/2602406893 2024-09-21T09:22:15.047 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.043+0000 7f084e7fc640 1 -- 172.21.15.78:0/434505038 --> v1:172.21.15.78:6812/3442117044 -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7f0868125ab0 con 0x7f0868063aa0 2024-09-21T09:22:15.048 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.042+0000 7fb26095f640 1 -- 172.21.15.78:0/4247053430 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fb25c0707c0 con 0x7fb25c12f2f0 2024-09-21T09:22:15.048 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.042+0000 7fb26095f640 1 -- 172.21.15.78:0/4247053430 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fb25c0983c0 con 0x7fb25c12f2f0 2024-09-21T09:22:15.048 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.043+0000 7fb24affd640 1 -- 172.21.15.78:0/4247053430 <== mon.0 v2:172.21.15.78:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fb250011920 con 0x7fb25c12f2f0 2024-09-21T09:22:15.048 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.044+0000 7fb24affd640 1 -- 172.21.15.78:0/4247053430 <== mon.0 v2:172.21.15.78:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb250011df0 con 0x7fb25c12f2f0 2024-09-21T09:22:15.048 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.044+0000 7fb26095f640 1 -- 172.21.15.78:0/4247053430 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_get_version(what=osdmap handle=1) -- 0x7fb240001030 con 0x7fb25c12f2f0 2024-09-21T09:22:15.050 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.045+0000 7fb24affd640 1 -- 172.21.15.78:0/4247053430 <== mon.0 v2:172.21.15.78:3300/0 4 ==== mgrmap(e 6) ==== 55351+0+0 (secure 0 0 0) 0x7fb250018020 con 0x7fb25c12f2f0 2024-09-21T09:22:15.050 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.046+0000 7fb25a59c640 1 -- 172.21.15.78:0/4247053430 >> v1:172.21.15.78:6816/2602406893 conn(0x7fb23c0476c0 legacy=0x7fb23c049b80 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.78:6816/2602406893 2024-09-21T09:22:15.050 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.046+0000 7fb24affd640 1 -- 172.21.15.78:0/4247053430 <== mon.0 v2:172.21.15.78:3300/0 5 ==== osd_map(19..19 src has 1..19) ==== 2812+0+0 (secure 0 0 0) 0x7fb25005d160 con 0x7fb25c12f2f0 2024-09-21T09:22:15.050 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.047+0000 7fb24affd640 1 -- 172.21.15.78:0/4247053430 --> v1:172.21.15.78:6800/3190796132 -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7fb23c051a60 con 0x7fb23c04ddb0 2024-09-21T09:22:15.050 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.047+0000 7fb24affd640 1 -- 172.21.15.78:0/4247053430 <== mon.0 v2:172.21.15.78:3300/0 6 ==== mon_get_version_reply(handle=1 version=19) ==== 24+0+0 (secure 0 0 0) 0x7fb25005d550 con 0x7fb25c12f2f0 2024-09-21T09:22:15.050 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.045+0000 7f0864ff9640 1 -- 172.21.15.78:0/434505038 <== osd.2 v1:172.21.15.78:6812/3442117044 1 ==== command_reply(tid 1: 0 ) ==== 8+0+26118 (unknown 0 0 1686565167) 0x7f0868125ab0 con 0x7f0868063aa0 2024-09-21T09:22:15.050 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.048+0000 7fb24affd640 1 -- 172.21.15.78:0/4247053430 <== osd.1 v1:172.21.15.78:6800/3190796132 1 ==== command_reply(tid 1: 0 ) ==== 8+0+26118 (unknown 0 0 1686565167) 0x7fb23c051a60 con 0x7fb23c04ddb0 2024-09-21T09:22:15.052 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.050+0000 7f4a2c16a640 1 -- 172.21.15.78:0/1105942921 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f4a241388a0 msgr2=0x7f4a2413ac90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:15.053 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.050+0000 7f4a2c16a640 1 --2- 172.21.15.78:0/1105942921 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f4a241388a0 0x7f4a2413ac90 secure :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0x7f4a14009860 tx=0x7f4a1402a410 comp rx=0 tx=0).stop 2024-09-21T09:22:15.053 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.051+0000 7f4a2c16a640 1 -- 172.21.15.78:0/1105942921 shutdown_connections 2024-09-21T09:22:15.053 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.051+0000 7f4a2c16a640 1 --2- 172.21.15.78:0/1105942921 >> v2:172.21.15.78:3300/0 conn(0x7f4a24076360 0x7f4a24138360 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:15.053 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.051+0000 7f4a2c16a640 1 --2- 172.21.15.78:0/1105942921 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f4a241388a0 0x7f4a2413ac90 unknown :-1 s=CLOSED pgs=14 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:15.053 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.051+0000 7f4a2c16a640 1 -- 172.21.15.78:0/1105942921 >> 172.21.15.78:0/1105942921 conn(0x7f4a2409dde0 msgr2=0x7f4a2409e1f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:15.053 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.051+0000 7f4a2c16a640 1 -- 172.21.15.78:0/1105942921 shutdown_connections 2024-09-21T09:22:15.054 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.051+0000 7f4a2c16a640 1 -- 172.21.15.78:0/1105942921 wait complete. 2024-09-21T09:22:15.057 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.052+0000 7f4a2c16a640 1 Processor -- start 2024-09-21T09:22:15.057 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.052+0000 7f4a2c16a640 1 -- start start 2024-09-21T09:22:15.057 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.053+0000 7f4a2c16a640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f4a24073bf0 0x7f4a2412eb40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:15.057 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.053+0000 7f4a2c16a640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f4a24076360 0x7f4a2412f080 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:15.058 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.053+0000 7f4a2c16a640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f4a241388a0 0x7f4a2412f5c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:15.058 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.053+0000 7f4a2c16a640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7f4a2413bd70 con 0x7f4a241388a0 2024-09-21T09:22:15.058 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.053+0000 7f4a2c16a640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7f4a2413bbf0 con 0x7f4a24076360 2024-09-21T09:22:15.058 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.053+0000 7f4a2c16a640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7f4a2413bef0 con 0x7f4a24073bf0 2024-09-21T09:22:15.058 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.053+0000 7f4a29705640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f4a24076360 0x7f4a2412f080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:15.058 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.053+0000 7f4a29705640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f4a24076360 0x7f4a2412f080 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3301/0 says I am v2:172.21.15.78:60612/0 (socket says 172.21.15.78:60612) 2024-09-21T09:22:15.058 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.053+0000 7f4a29705640 1 -- 172.21.15.78:0/1375056417 learned_addr learned my addr 172.21.15.78:0/1375056417 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:15.058 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.053+0000 7f4a2a707640 1 --2- 172.21.15.78:0/1375056417 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f4a241388a0 0x7f4a2412f5c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:15.058 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.053+0000 7f4a29f06640 1 --2- 172.21.15.78:0/1375056417 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f4a24073bf0 0x7f4a2412eb40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:15.058 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.054+0000 7f4a2a707640 1 -- 172.21.15.78:0/1375056417 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f4a24073bf0 msgr2=0x7f4a2412eb40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:15.058 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.054+0000 7f4a2a707640 1 --2- 172.21.15.78:0/1375056417 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f4a24073bf0 0x7f4a2412eb40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:15.058 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.054+0000 7f4a2a707640 1 -- 172.21.15.78:0/1375056417 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f4a24076360 msgr2=0x7f4a2412f080 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:15.058 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.054+0000 7f4a2a707640 1 --2- 172.21.15.78:0/1375056417 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f4a24076360 0x7f4a2412f080 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:15.059 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.054+0000 7f4a2a707640 1 -- 172.21.15.78:0/1375056417 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f4a240952e0 con 0x7f4a241388a0 2024-09-21T09:22:15.059 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.054+0000 7f4a29f06640 1 --2- 172.21.15.78:0/1375056417 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f4a24073bf0 0x7f4a2412eb40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-09-21T09:22:15.059 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.054+0000 7f4a2a707640 1 --2- 172.21.15.78:0/1375056417 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f4a241388a0 0x7f4a2412f5c0 secure :-1 s=READY pgs=32 cs=0 l=1 rev1=1 crypto rx=0x7f4a14031000 tx=0x7f4a14003b90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:15.059 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.055+0000 7f4a1a7fc640 1 -- 172.21.15.78:0/1375056417 <== mon.0 v2:172.21.15.78:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4a14005020 con 0x7f4a241388a0 2024-09-21T09:22:15.060 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.055+0000 7f4a2c16a640 1 -- 172.21.15.78:0/1375056417 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f4a240969e0 con 0x7f4a241388a0 2024-09-21T09:22:15.060 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.055+0000 7f4a2c16a640 1 -- 172.21.15.78:0/1375056417 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f4a240955a0 con 0x7f4a241388a0 2024-09-21T09:22:15.060 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.056+0000 7f4a1a7fc640 1 -- 172.21.15.78:0/1375056417 <== mon.0 v2:172.21.15.78:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f4a140052b0 con 0x7f4a241388a0 2024-09-21T09:22:15.060 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.057+0000 7f4a1a7fc640 1 -- 172.21.15.78:0/1375056417 <== mon.0 v2:172.21.15.78:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4a140047a0 con 0x7f4a241388a0 2024-09-21T09:22:15.060 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.057+0000 7f4a1a7fc640 1 -- 172.21.15.78:0/1375056417 <== mon.0 v2:172.21.15.78:3300/0 4 ==== mgrmap(e 6) ==== 55351+0+0 (secure 0 0 0) 0x7f4a14039040 con 0x7f4a241388a0 2024-09-21T09:22:15.060 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.058+0000 7f4a29f06640 1 -- 172.21.15.78:0/1375056417 >> v1:172.21.15.78:6816/2602406893 conn(0x7f4a080476c0 legacy=0x7f4a08049b80 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.78:6816/2602406893 2024-09-21T09:22:15.060 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.058+0000 7f4a1a7fc640 1 -- 172.21.15.78:0/1375056417 <== mon.0 v2:172.21.15.78:3300/0 5 ==== osd_map(19..19 src has 1..19) ==== 2812+0+0 (secure 0 0 0) 0x7f4a1407df70 con 0x7f4a241388a0 2024-09-21T09:22:15.061 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.059+0000 7f4a2c16a640 1 -- 172.21.15.78:0/1375056417 --> v1:172.21.15.78:6802/4276488585 -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7f4a0c005420 con 0x7f4a0c001630 2024-09-21T09:22:15.061 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.059+0000 7fba35ffb640 1 -- 172.21.15.78:0/2207083386 --> v1:172.21.15.78:6808/1449157914 -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7fba401314c0 con 0x7fba2004dc70 2024-09-21T09:22:15.062 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.059+0000 7fba3cff9640 1 -- 172.21.15.78:0/2207083386 <== osd.0 v1:172.21.15.78:6808/1449157914 2 ==== command_reply(tid 2: 0 ) ==== 8+0+11 (unknown 0 0 3077695196) 0x7fba401314c0 con 0x7fba2004dc70 2024-09-21T09:22:15.062 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.060+0000 7f4a1a7fc640 1 -- 172.21.15.78:0/1375056417 <== osd.3 v1:172.21.15.78:6802/4276488585 1 ==== command_reply(tid 1: 0 ) ==== 8+0+26118 (unknown 0 0 1686565167) 0x7f4a0c005420 con 0x7f4a0c001630 2024-09-21T09:22:15.062 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.060+0000 7fba35ffb640 1 -- 172.21.15.78:0/2207083386 >> v1:172.21.15.78:6808/1449157914 conn(0x7fba2004dc70 legacy=0x7fba200500b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:15.063 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.061+0000 7fba35ffb640 1 -- 172.21.15.78:0/2207083386 >> v1:172.21.15.78:6816/2602406893 conn(0x7fba200475f0 legacy=0x7fba20049ab0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-21T09:22:15.063 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.061+0000 7fba35ffb640 1 -- 172.21.15.78:0/2207083386 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fba400973f0 msgr2=0x7fba40130620 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:15.063 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.061+0000 7fba35ffb640 1 --2- 172.21.15.78:0/2207083386 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fba400973f0 0x7fba40130620 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7fba38009b30 tx=0x7fba380315b0 comp rx=0 tx=0).stop 2024-09-21T09:22:15.063 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.061+0000 7fba3ffff640 1 -- 172.21.15.78:0/2207083386 reap_dead start 2024-09-21T09:22:15.063 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.061+0000 7fba35ffb640 1 -- 172.21.15.78:0/2207083386 shutdown_connections 2024-09-21T09:22:15.064 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.061+0000 7fba35ffb640 1 -- 172.21.15.78:0/2207083386 >> 172.21.15.78:0/2207083386 conn(0x7fba4009c470 msgr2=0x7fba40099f80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:15.064 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.061+0000 7fba35ffb640 1 -- 172.21.15.78:0/2207083386 shutdown_connections 2024-09-21T09:22:15.064 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.061+0000 7fba35ffb640 1 -- 172.21.15.78:0/2207083386 wait complete. 2024-09-21T09:22:15.072 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.070+0000 7f084e7fc640 1 -- 172.21.15.78:0/434505038 --> v1:172.21.15.78:6812/3442117044 -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7f08681c9240 con 0x7f0868063aa0 2024-09-21T09:22:15.072 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.070+0000 7f0864ff9640 1 -- 172.21.15.78:0/434505038 <== osd.2 v1:172.21.15.78:6812/3442117044 2 ==== command_reply(tid 2: 0 ) ==== 8+0+11 (unknown 0 0 2446328116) 0x7f08681c9240 con 0x7f0868063aa0 2024-09-21T09:22:15.073 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.071+0000 7f084e7fc640 1 -- 172.21.15.78:0/434505038 >> v1:172.21.15.78:6812/3442117044 conn(0x7f0868063aa0 legacy=0x7f0868063e80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:15.074 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.071+0000 7f084e7fc640 1 -- 172.21.15.78:0/434505038 >> v1:172.21.15.78:6816/2602406893 conn(0x7f08540476f0 legacy=0x7f0854049bb0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-21T09:22:15.074 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.071+0000 7f084e7fc640 1 -- 172.21.15.78:0/434505038 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f086808c4c0 msgr2=0x7f086808c900 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:15.074 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.071+0000 7f084e7fc640 1 --2- 172.21.15.78:0/434505038 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f086808c4c0 0x7f086808c900 secure :-1 s=READY pgs=29 cs=0 l=1 rev1=1 crypto rx=0x7f08680704e0 tx=0x7f085c00b730 comp rx=0 tx=0).stop 2024-09-21T09:22:15.074 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.071+0000 7f086c896640 1 -- 172.21.15.78:0/434505038 reap_dead start 2024-09-21T09:22:15.074 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.071+0000 7f084e7fc640 1 -- 172.21.15.78:0/434505038 shutdown_connections 2024-09-21T09:22:15.074 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.071+0000 7f084e7fc640 1 -- 172.21.15.78:0/434505038 >> 172.21.15.78:0/434505038 conn(0x7f086809dde0 msgr2=0x7f086812dd30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:15.074 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.071+0000 7f084e7fc640 1 -- 172.21.15.78:0/434505038 shutdown_connections 2024-09-21T09:22:15.074 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.071+0000 7f084e7fc640 1 -- 172.21.15.78:0/434505038 wait complete. 2024-09-21T09:22:15.076 INFO:teuthology.orchestra.run.smithi078.stdout:38654705683 2024-09-21T09:22:15.077 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.0 2024-09-21T09:22:15.080 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.077+0000 7fb26095f640 1 -- 172.21.15.78:0/4247053430 --> v1:172.21.15.78:6800/3190796132 -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7fb240002dd0 con 0x7fb23c04ddb0 2024-09-21T09:22:15.080 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.077+0000 7fb24affd640 1 -- 172.21.15.78:0/4247053430 <== osd.1 v1:172.21.15.78:6800/3190796132 2 ==== command_reply(tid 2: 0 ) ==== 8+0+11 (unknown 0 0 3077695196) 0x7fb240002dd0 con 0x7fb23c04ddb0 2024-09-21T09:22:15.081 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.078+0000 7fb26095f640 1 -- 172.21.15.78:0/4247053430 >> v1:172.21.15.78:6800/3190796132 conn(0x7fb23c04ddb0 legacy=0x7fb23c0501f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:15.081 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.078+0000 7fb26095f640 1 -- 172.21.15.78:0/4247053430 >> v1:172.21.15.78:6816/2602406893 conn(0x7fb23c0476c0 legacy=0x7fb23c049b80 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-21T09:22:15.081 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.078+0000 7fb26095f640 1 -- 172.21.15.78:0/4247053430 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fb25c12f2f0 msgr2=0x7fb25c0961e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:15.081 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.078+0000 7fb26095f640 1 --2- 172.21.15.78:0/4247053430 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fb25c12f2f0 0x7fb25c0961e0 secure :-1 s=READY pgs=30 cs=0 l=1 rev1=1 crypto rx=0x7fb25000b3e0 tx=0x7fb25000b8a0 comp rx=0 tx=0).stop 2024-09-21T09:22:15.081 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.078+0000 7fb25ad9d640 1 -- 172.21.15.78:0/4247053430 reap_dead start 2024-09-21T09:22:15.081 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.078+0000 7fb26095f640 1 -- 172.21.15.78:0/4247053430 shutdown_connections 2024-09-21T09:22:15.081 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.078+0000 7fb26095f640 1 -- 172.21.15.78:0/4247053430 >> 172.21.15.78:0/4247053430 conn(0x7fb25c09e490 msgr2=0x7fb25c098a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:15.081 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.078+0000 7fb26095f640 1 -- 172.21.15.78:0/4247053430 shutdown_connections 2024-09-21T09:22:15.081 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.078+0000 7fb26095f640 1 -- 172.21.15.78:0/4247053430 wait complete. 2024-09-21T09:22:15.088 INFO:teuthology.orchestra.run.smithi078.stdout:38654705685 2024-09-21T09:22:15.088 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.2 2024-09-21T09:22:15.090 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.088+0000 7f4a2c16a640 1 -- 172.21.15.78:0/1375056417 --> v1:172.21.15.78:6802/4276488585 -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7f4a0c007190 con 0x7f4a0c001630 2024-09-21T09:22:15.090 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.088+0000 7f4a1a7fc640 1 -- 172.21.15.78:0/1375056417 <== osd.3 v1:172.21.15.78:6802/4276488585 2 ==== command_reply(tid 2: 0 ) ==== 8+0+11 (unknown 0 0 3077695196) 0x7f4a0c007190 con 0x7f4a0c001630 2024-09-21T09:22:15.092 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.089+0000 7f4a2c16a640 1 -- 172.21.15.78:0/1375056417 >> v1:172.21.15.78:6802/4276488585 conn(0x7f4a0c001630 legacy=0x7f4a0c003af0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:15.092 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.089+0000 7f4a2c16a640 1 -- 172.21.15.78:0/1375056417 >> v1:172.21.15.78:6816/2602406893 conn(0x7f4a080476c0 legacy=0x7f4a08049b80 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-21T09:22:15.092 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.089+0000 7f4a2c16a640 1 -- 172.21.15.78:0/1375056417 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f4a241388a0 msgr2=0x7f4a2412f5c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:15.092 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.089+0000 7f4a2c16a640 1 --2- 172.21.15.78:0/1375056417 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f4a241388a0 0x7f4a2412f5c0 secure :-1 s=READY pgs=32 cs=0 l=1 rev1=1 crypto rx=0x7f4a14031000 tx=0x7f4a14003b90 comp rx=0 tx=0).stop 2024-09-21T09:22:15.092 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.089+0000 7f4a2a707640 1 -- 172.21.15.78:0/1375056417 reap_dead start 2024-09-21T09:22:15.092 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.089+0000 7f4a2c16a640 1 -- 172.21.15.78:0/1375056417 shutdown_connections 2024-09-21T09:22:15.092 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.089+0000 7f4a2c16a640 1 -- 172.21.15.78:0/1375056417 >> 172.21.15.78:0/1375056417 conn(0x7f4a2409dde0 msgr2=0x7f4a2409abe0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:15.092 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.089+0000 7f4a2c16a640 1 -- 172.21.15.78:0/1375056417 shutdown_connections 2024-09-21T09:22:15.093 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.089+0000 7f4a2c16a640 1 -- 172.21.15.78:0/1375056417 wait complete. 2024-09-21T09:22:15.100 INFO:teuthology.orchestra.run.smithi078.stdout:38654705683 2024-09-21T09:22:15.100 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.1 2024-09-21T09:22:15.107 INFO:teuthology.orchestra.run.smithi078.stdout:38654705683 2024-09-21T09:22:15.108 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.3 2024-09-21T09:22:15.227 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.224+0000 7eff4e108640 1 -- 172.21.15.78:0/1976742037 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7eff480961b0 msgr2=0x7eff48096660 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:15.227 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.224+0000 7eff4e108640 1 --2- 172.21.15.78:0/1976742037 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7eff480961b0 0x7eff48096660 secure :-1 s=READY pgs=33 cs=0 l=1 rev1=1 crypto rx=0x7eff40005fd0 tx=0x7eff4002a6e0 comp rx=0 tx=0).stop 2024-09-21T09:22:15.227 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.224+0000 7eff4e108640 1 -- 172.21.15.78:0/1976742037 shutdown_connections 2024-09-21T09:22:15.227 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.224+0000 7eff4e108640 1 --2- 172.21.15.78:0/1976742037 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7eff48094c90 0x7eff48095c70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:15.227 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.224+0000 7eff4e108640 1 --2- 172.21.15.78:0/1976742037 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7eff480961b0 0x7eff48096660 unknown :-1 s=CLOSED pgs=33 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:15.227 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.224+0000 7eff4e108640 1 --2- 172.21.15.78:0/1976742037 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7eff48094370 0x7eff48094750 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:15.227 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.224+0000 7eff4e108640 1 -- 172.21.15.78:0/1976742037 >> 172.21.15.78:0/1976742037 conn(0x7eff4809e610 msgr2=0x7eff4809ea20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:15.228 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.224+0000 7eff4e108640 1 -- 172.21.15.78:0/1976742037 shutdown_connections 2024-09-21T09:22:15.228 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.224+0000 7eff4e108640 1 -- 172.21.15.78:0/1976742037 wait complete. 2024-09-21T09:22:15.228 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.226+0000 7eff4e108640 1 Processor -- start 2024-09-21T09:22:15.228 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.226+0000 7eff4e108640 1 -- start start 2024-09-21T09:22:15.229 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.226+0000 7eff4e108640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7eff48094370 0x7eff4812d640 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:15.229 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.226+0000 7eff4e108640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7eff48094c90 0x7eff4812db80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:15.229 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.226+0000 7eff4e108640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7eff480961b0 0x7eff4812e0c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:15.230 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.226+0000 7eff4e108640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7eff4806e900 con 0x7eff48094c90 2024-09-21T09:22:15.230 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.226+0000 7eff4e108640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7eff4806e780 con 0x7eff480961b0 2024-09-21T09:22:15.230 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.226+0000 7eff4e108640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7eff4806ea80 con 0x7eff48094370 2024-09-21T09:22:15.230 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.227+0000 7eff477fe640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7eff48094370 0x7eff4812d640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:15.230 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.227+0000 7eff46ffd640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7eff48094c90 0x7eff4812db80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:15.230 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.227+0000 7eff47fff640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7eff480961b0 0x7eff4812e0c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:15.230 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.227+0000 7eff46ffd640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7eff48094c90 0x7eff4812db80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3300/0 says I am v2:172.21.15.78:51138/0 (socket says 172.21.15.78:51138) 2024-09-21T09:22:15.232 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.227+0000 7eff46ffd640 1 -- 172.21.15.78:0/3868308737 learned_addr learned my addr 172.21.15.78:0/3868308737 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:15.232 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.228+0000 7eff46ffd640 1 -- 172.21.15.78:0/3868308737 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7eff48094370 msgr2=0x7eff4812d640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:15.232 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.228+0000 7eff46ffd640 1 --2- 172.21.15.78:0/3868308737 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7eff48094370 0x7eff4812d640 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:15.232 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.228+0000 7eff46ffd640 1 -- 172.21.15.78:0/3868308737 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7eff480961b0 msgr2=0x7eff4812e0c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:15.232 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.228+0000 7eff46ffd640 1 --2- 172.21.15.78:0/3868308737 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7eff480961b0 0x7eff4812e0c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:15.232 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.228+0000 7eff46ffd640 1 -- 172.21.15.78:0/3868308737 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7eff48130070 con 0x7eff48094c90 2024-09-21T09:22:15.232 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.228+0000 7eff47fff640 1 --2- 172.21.15.78:0/3868308737 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7eff480961b0 0x7eff4812e0c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-09-21T09:22:15.232 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.228+0000 7eff477fe640 1 --2- 172.21.15.78:0/3868308737 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7eff48094370 0x7eff4812d640 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-09-21T09:22:15.232 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.228+0000 7eff46ffd640 1 --2- 172.21.15.78:0/3868308737 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7eff48094c90 0x7eff4812db80 secure :-1 s=READY pgs=34 cs=0 l=1 rev1=1 crypto rx=0x7eff3c00ed30 tx=0x7eff3c00c6a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:15.232 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.228+0000 7eff2ffff640 1 -- 172.21.15.78:0/3868308737 <== mon.0 v2:172.21.15.78:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7eff3c009e10 con 0x7eff48094c90 2024-09-21T09:22:15.232 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.229+0000 7eff2ffff640 1 -- 172.21.15.78:0/3868308737 <== mon.0 v2:172.21.15.78:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7eff3c004020 con 0x7eff48094c90 2024-09-21T09:22:15.232 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.229+0000 7eff4e108640 1 -- 172.21.15.78:0/3868308737 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7eff48131b40 con 0x7eff48094c90 2024-09-21T09:22:15.233 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.229+0000 7eff2ffff640 1 -- 172.21.15.78:0/3868308737 <== mon.0 v2:172.21.15.78:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7eff3c009e10 con 0x7eff48094c90 2024-09-21T09:22:15.233 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.229+0000 7eff4e108640 1 -- 172.21.15.78:0/3868308737 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7eff4812e600 con 0x7eff48094c90 2024-09-21T09:22:15.233 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.231+0000 7eff4e108640 1 -- 172.21.15.78:0/3868308737 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7eff4806f010 con 0x7eff48094c90 2024-09-21T09:22:15.235 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.232+0000 7eff2ffff640 1 -- 172.21.15.78:0/3868308737 <== mon.0 v2:172.21.15.78:3300/0 4 ==== mgrmap(e 6) ==== 55351+0+0 (secure 0 0 0) 0x7eff3c0041d0 con 0x7eff48094c90 2024-09-21T09:22:15.236 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.233+0000 7eff2ffff640 1 -- 172.21.15.78:0/3868308737 <== mon.0 v2:172.21.15.78:3300/0 5 ==== osd_map(19..19 src has 1..19) ==== 2812+0+0 (secure 0 0 0) 0x7eff3c05d530 con 0x7eff48094c90 2024-09-21T09:22:15.236 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.233+0000 7eff477fe640 1 -- 172.21.15.78:0/3868308737 >> v1:172.21.15.78:6816/2602406893 conn(0x7eff340477f0 legacy=0x7eff34049cb0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.78:6816/2602406893 2024-09-21T09:22:15.240 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.238+0000 7eff2ffff640 1 -- 172.21.15.78:0/3868308737 <== mon.0 v2:172.21.15.78:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+148921 (secure 0 0 0) 0x7eff48130070 con 0x7eff48094c90 2024-09-21T09:22:15.256 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.253+0000 7fab9f40e640 1 -- 172.21.15.78:0/259509139 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fab90018ff0 msgr2=0x7fab900af5c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:15.256 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.253+0000 7fab9f40e640 1 --2- 172.21.15.78:0/259509139 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fab90018ff0 0x7fab900af5c0 secure :-1 s=READY pgs=36 cs=0 l=1 rev1=1 crypto rx=0x7fab88005fd0 tx=0x7fab8802a500 comp rx=0 tx=0).stop 2024-09-21T09:22:15.256 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.253+0000 7fab9f40e640 1 -- 172.21.15.78:0/259509139 shutdown_connections 2024-09-21T09:22:15.256 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.253+0000 7fab9f40e640 1 --2- 172.21.15.78:0/259509139 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fab90018ff0 0x7fab900af5c0 secure :-1 s=CLOSED pgs=36 cs=0 l=1 rev1=1 crypto rx=0x7fab88005fd0 tx=0x7fab8802a500 comp rx=0 tx=0).stop 2024-09-21T09:22:15.256 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.253+0000 7fab9f40e640 1 --2- 172.21.15.78:0/259509139 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fab900186d0 0x7fab90018ab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:15.256 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.253+0000 7fab9f40e640 1 -- 172.21.15.78:0/259509139 >> 172.21.15.78:0/259509139 conn(0x7fab90005c50 msgr2=0x7fab90006060 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:15.256 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.254+0000 7fab9f40e640 1 -- 172.21.15.78:0/259509139 shutdown_connections 2024-09-21T09:22:15.256 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.254+0000 7fab9f40e640 1 -- 172.21.15.78:0/259509139 wait complete. 2024-09-21T09:22:15.257 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.255+0000 7fab9f40e640 1 Processor -- start 2024-09-21T09:22:15.257 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.255+0000 7fab9f40e640 1 -- start start 2024-09-21T09:22:15.258 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.256+0000 7fab9f40e640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fab900186d0 0x7fab90012c20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:15.258 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.256+0000 7fab9f40e640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fab900afb00 0x7fab90013160 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:15.258 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.256+0000 7fab9f40e640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fab900136a0 0x7fab900aa620 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:15.259 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.256+0000 7fab9f40e640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7fab90018050 con 0x7fab900afb00 2024-09-21T09:22:15.259 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.256+0000 7fab9f40e640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7fab90017ed0 con 0x7fab900136a0 2024-09-21T09:22:15.259 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.256+0000 7fab9f40e640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7fab900181d0 con 0x7fab900186d0 2024-09-21T09:22:15.259 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.256+0000 7fab9dc0b640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fab900afb00 0x7fab90013160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:15.259 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.256+0000 7fab9dc0b640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fab900afb00 0x7fab90013160 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3300/0 says I am v2:172.21.15.78:51158/0 (socket says 172.21.15.78:51158) 2024-09-21T09:22:15.259 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.256+0000 7fab9dc0b640 1 -- 172.21.15.78:0/1688357588 learned_addr learned my addr 172.21.15.78:0/1688357588 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:15.260 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.257+0000 7fab9dc0b640 1 -- 172.21.15.78:0/1688357588 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fab900186d0 msgr2=0x7fab90012c20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:15.260 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.257+0000 7fab9ec0d640 1 --2- 172.21.15.78:0/1688357588 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fab900136a0 0x7fab900aa620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:15.261 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.257+0000 7fab9e40c640 1 --2- 172.21.15.78:0/1688357588 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fab900186d0 0x7fab90012c20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:15.261 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.257+0000 7fab9dc0b640 1 --2- 172.21.15.78:0/1688357588 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fab900186d0 0x7fab90012c20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:15.261 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.257+0000 7fab9dc0b640 1 -- 172.21.15.78:0/1688357588 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fab900136a0 msgr2=0x7fab900aa620 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:15.261 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.257+0000 7fab9dc0b640 1 --2- 172.21.15.78:0/1688357588 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fab900136a0 0x7fab900aa620 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:15.261 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.257+0000 7fab9dc0b640 1 -- 172.21.15.78:0/1688357588 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fab900b09d0 con 0x7fab900afb00 2024-09-21T09:22:15.261 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.257+0000 7fab9e40c640 1 --2- 172.21.15.78:0/1688357588 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fab900186d0 0x7fab90012c20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-09-21T09:22:15.261 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.257+0000 7fab9ec0d640 1 --2- 172.21.15.78:0/1688357588 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fab900136a0 0x7fab900aa620 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-09-21T09:22:15.261 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.257+0000 7fab9dc0b640 1 --2- 172.21.15.78:0/1688357588 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fab900afb00 0x7fab90013160 secure :-1 s=READY pgs=37 cs=0 l=1 rev1=1 crypto rx=0x7fab880098a0 tx=0x7fab88004a50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:15.261 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.258+0000 7fab8effd640 1 -- 172.21.15.78:0/1688357588 <== mon.0 v2:172.21.15.78:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fab8803f020 con 0x7fab900afb00 2024-09-21T09:22:15.261 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.258+0000 7fab8effd640 1 -- 172.21.15.78:0/1688357588 <== mon.0 v2:172.21.15.78:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fab880035f0 con 0x7fab900afb00 2024-09-21T09:22:15.261 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.258+0000 7fab9f40e640 1 -- 172.21.15.78:0/1688357588 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fab900ac680 con 0x7fab900afb00 2024-09-21T09:22:15.261 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.258+0000 7fab9f40e640 1 -- 172.21.15.78:0/1688357588 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fab900aab60 con 0x7fab900afb00 2024-09-21T09:22:15.262 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.258+0000 7fab8effd640 1 -- 172.21.15.78:0/1688357588 <== mon.0 v2:172.21.15.78:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fab88035070 con 0x7fab900afb00 2024-09-21T09:22:15.262 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.259+0000 7fab9f40e640 1 -- 172.21.15.78:0/1688357588 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fab80005290 con 0x7fab900afb00 2024-09-21T09:22:15.264 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.261+0000 7fab8effd640 1 -- 172.21.15.78:0/1688357588 <== mon.0 v2:172.21.15.78:3300/0 4 ==== mgrmap(e 6) ==== 55351+0+0 (secure 0 0 0) 0x7fab88042020 con 0x7fab900afb00 2024-09-21T09:22:15.264 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.261+0000 7fab8effd640 1 -- 172.21.15.78:0/1688357588 <== mon.0 v2:172.21.15.78:3300/0 5 ==== osd_map(19..19 src has 1..19) ==== 2812+0+0 (secure 0 0 0) 0x7fab8807da40 con 0x7fab900afb00 2024-09-21T09:22:15.264 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.262+0000 7fab9e40c640 1 -- 172.21.15.78:0/1688357588 >> v1:172.21.15.78:6816/2602406893 conn(0x7fab7c0475f0 legacy=0x7fab7c049ab0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.78:6816/2602406893 2024-09-21T09:22:15.269 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.267+0000 7fab8effd640 1 -- 172.21.15.78:0/1688357588 <== mon.0 v2:172.21.15.78:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+148921 (secure 0 0 0) 0x7fab88005140 con 0x7fab900afb00 2024-09-21T09:22:15.306 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.303+0000 7f9281627640 1 -- 172.21.15.78:0/1770601383 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f927c070640 msgr2=0x7f927c070a20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:15.307 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.303+0000 7f9281627640 1 --2- 172.21.15.78:0/1770601383 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f927c070640 0x7f927c070a20 secure :-1 s=READY pgs=39 cs=0 l=1 rev1=1 crypto rx=0x7f926c005fd0 tx=0x7f926c02a520 comp rx=0 tx=0).stop 2024-09-21T09:22:15.307 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.303+0000 7f9281627640 1 -- 172.21.15.78:0/1770601383 shutdown_connections 2024-09-21T09:22:15.307 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.303+0000 7f9281627640 1 --2- 172.21.15.78:0/1770601383 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f927c097740 0x7f927c133ff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:15.307 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.303+0000 7f9281627640 1 --2- 172.21.15.78:0/1770601383 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f927c070640 0x7f927c070a20 unknown :-1 s=CLOSED pgs=39 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:15.307 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.303+0000 7f9281627640 1 -- 172.21.15.78:0/1770601383 >> 172.21.15.78:0/1770601383 conn(0x7f927c09dde0 msgr2=0x7f927c09e1f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:15.307 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.303+0000 7f9281627640 1 -- 172.21.15.78:0/1770601383 shutdown_connections 2024-09-21T09:22:15.307 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.304+0000 7f9281627640 1 -- 172.21.15.78:0/1770601383 wait complete. 2024-09-21T09:22:15.309 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.305+0000 7f9281627640 1 Processor -- start 2024-09-21T09:22:15.309 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.305+0000 7f9281627640 1 -- start start 2024-09-21T09:22:15.309 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.306+0000 7f9281627640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f927c070640 0x7f927c1b2190 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:15.310 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.306+0000 7f9281627640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f927c070f60 0x7f927c1b26d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:15.310 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.306+0000 7f9281627640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f927c097740 0x7f927c1b2c10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:15.310 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.306+0000 7f9281627640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7f927c06f9a0 con 0x7f927c070640 2024-09-21T09:22:15.310 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.306+0000 7f9281627640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7f927c06f820 con 0x7f927c097740 2024-09-21T09:22:15.310 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.306+0000 7f9281627640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7f927c06fb20 con 0x7f927c070f60 2024-09-21T09:22:15.310 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.306+0000 7f927affd640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f927c070640 0x7f927c1b2190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:15.310 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.306+0000 7f927affd640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f927c070640 0x7f927c1b2190 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3300/0 says I am v2:172.21.15.78:51170/0 (socket says 172.21.15.78:51170) 2024-09-21T09:22:15.310 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.306+0000 7f927affd640 1 -- 172.21.15.78:0/2510468278 learned_addr learned my addr 172.21.15.78:0/2510468278 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:15.310 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.306+0000 7f927a7fc640 1 --2- 172.21.15.78:0/2510468278 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f927c070f60 0x7f927c1b26d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:15.310 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.306+0000 7f927b7fe640 1 --2- 172.21.15.78:0/2510468278 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f927c097740 0x7f927c1b2c10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:15.310 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.307+0000 7f927affd640 1 -- 172.21.15.78:0/2510468278 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f927c070f60 msgr2=0x7f927c1b26d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:15.310 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.307+0000 7f927affd640 1 --2- 172.21.15.78:0/2510468278 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f927c070f60 0x7f927c1b26d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:15.311 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.307+0000 7f927affd640 1 -- 172.21.15.78:0/2510468278 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f927c097740 msgr2=0x7f927c1b2c10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:15.311 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.307+0000 7f927affd640 1 --2- 172.21.15.78:0/2510468278 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f927c097740 0x7f927c1b2c10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:15.311 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.307+0000 7f927affd640 1 -- 172.21.15.78:0/2510468278 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f927c1b3620 con 0x7f927c070640 2024-09-21T09:22:15.311 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.307+0000 7f927a7fc640 1 --2- 172.21.15.78:0/2510468278 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f927c070f60 0x7f927c1b26d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-09-21T09:22:15.311 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.307+0000 7f927b7fe640 1 --2- 172.21.15.78:0/2510468278 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f927c097740 0x7f927c1b2c10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:22:15.312 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.308+0000 7f927affd640 1 --2- 172.21.15.78:0/2510468278 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f927c070640 0x7f927c1b2190 secure :-1 s=READY pgs=40 cs=0 l=1 rev1=1 crypto rx=0x7f926c006010 tx=0x7f926c003820 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:15.312 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.308+0000 7f92637fe640 1 -- 172.21.15.78:0/2510468278 <== mon.0 v2:172.21.15.78:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f926c03f020 con 0x7f927c070640 2024-09-21T09:22:15.312 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.308+0000 7f92637fe640 1 -- 172.21.15.78:0/2510468278 <== mon.0 v2:172.21.15.78:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f926c0047a0 con 0x7f927c070640 2024-09-21T09:22:15.312 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.308+0000 7f92637fe640 1 -- 172.21.15.78:0/2510468278 <== mon.0 v2:172.21.15.78:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f926c035070 con 0x7f927c070640 2024-09-21T09:22:15.312 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.308+0000 7f9281627640 1 -- 172.21.15.78:0/2510468278 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f927c1b5290 con 0x7f927c070640 2024-09-21T09:22:15.312 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.308+0000 7f9281627640 1 -- 172.21.15.78:0/2510468278 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f927c1b46e0 con 0x7f927c070640 2024-09-21T09:22:15.312 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.310+0000 7f9281627640 1 -- 172.21.15.78:0/2510468278 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f927c070a20 con 0x7f927c070640 2024-09-21T09:22:15.313 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.310+0000 7f92637fe640 1 -- 172.21.15.78:0/2510468278 <== mon.0 v2:172.21.15.78:3300/0 4 ==== mgrmap(e 6) ==== 55351+0+0 (secure 0 0 0) 0x7f926c042020 con 0x7f927c070640 2024-09-21T09:22:15.313 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.311+0000 7f92637fe640 1 -- 172.21.15.78:0/2510468278 <== mon.0 v2:172.21.15.78:3300/0 5 ==== osd_map(19..19 src has 1..19) ==== 2812+0+0 (secure 0 0 0) 0x7f926c07d7b0 con 0x7f927c070640 2024-09-21T09:22:15.314 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.312+0000 7f927a7fc640 1 -- 172.21.15.78:0/2510468278 >> v1:172.21.15.78:6816/2602406893 conn(0x7f92680476c0 legacy=0x7f9268049b80 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.78:6816/2602406893 2024-09-21T09:22:15.319 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.316+0000 7fa72767a640 1 -- 172.21.15.78:0/87716299 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fa720073fd0 msgr2=0x7fa72013ab10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:15.319 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.316+0000 7fa72767a640 1 --2- 172.21.15.78:0/87716299 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fa720073fd0 0x7fa72013ab10 secure :-1 s=READY pgs=20 cs=0 l=1 rev1=1 crypto rx=0x7fa718009be0 tx=0x7fa71802a740 comp rx=0 tx=0).stop 2024-09-21T09:22:15.319 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.317+0000 7fa72767a640 1 -- 172.21.15.78:0/87716299 shutdown_connections 2024-09-21T09:22:15.319 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.317+0000 7fa72767a640 1 --2- 172.21.15.78:0/87716299 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fa720073fd0 0x7fa72013ab10 unknown :-1 s=CLOSED pgs=20 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:15.319 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.317+0000 7fa72767a640 1 --2- 172.21.15.78:0/87716299 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fa72012ed20 0x7fa7201385c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:15.320 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.317+0000 7fa72767a640 1 -- 172.21.15.78:0/87716299 >> 172.21.15.78:0/87716299 conn(0x7fa72009e610 msgr2=0x7fa72009ea20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:15.320 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.317+0000 7fa72767a640 1 -- 172.21.15.78:0/87716299 shutdown_connections 2024-09-21T09:22:15.320 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.317+0000 7f92637fe640 1 -- 172.21.15.78:0/2510468278 <== mon.0 v2:172.21.15.78:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+148921 (secure 0 0 0) 0x7f926c049030 con 0x7f927c070640 2024-09-21T09:22:15.320 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.317+0000 7fa72767a640 1 -- 172.21.15.78:0/87716299 wait complete. 2024-09-21T09:22:15.321 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.319+0000 7fa72767a640 1 Processor -- start 2024-09-21T09:22:15.321 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.319+0000 7fa72767a640 1 -- start start 2024-09-21T09:22:15.321 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.319+0000 7fa72767a640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fa720073fd0 0x7fa7201c9320 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:15.322 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.319+0000 7fa72767a640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fa72012ed20 0x7fa7201c9860 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:15.322 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.320+0000 7fa72767a640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fa72012f490 0x7fa720070610 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:15.323 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.320+0000 7fa725416640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fa720073fd0 0x7fa7201c9320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:15.323 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.320+0000 7fa725416640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fa720073fd0 0x7fa7201c9320 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3301/0 says I am v2:172.21.15.78:60684/0 (socket says 172.21.15.78:60684) 2024-09-21T09:22:15.323 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.320+0000 7fa725416640 1 -- 172.21.15.78:0/1839191392 learned_addr learned my addr 172.21.15.78:0/1839191392 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:15.323 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.320+0000 7fa724c15640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fa72012ed20 0x7fa7201c9860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:15.323 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.320+0000 7fa725c17640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fa72012f490 0x7fa720070610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:15.323 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.320+0000 7fa72767a640 1 -- 172.21.15.78:0/1839191392 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7fa72013bc60 con 0x7fa72012f490 2024-09-21T09:22:15.323 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.320+0000 7fa72767a640 1 -- 172.21.15.78:0/1839191392 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7fa72013bae0 con 0x7fa720073fd0 2024-09-21T09:22:15.323 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.320+0000 7fa72767a640 1 -- 172.21.15.78:0/1839191392 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7fa72013bde0 con 0x7fa72012ed20 2024-09-21T09:22:15.324 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.321+0000 7fa724c15640 1 -- 172.21.15.78:0/1839191392 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fa720073fd0 msgr2=0x7fa7201c9320 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:15.325 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.321+0000 7fa724c15640 1 --2- 172.21.15.78:0/1839191392 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fa720073fd0 0x7fa7201c9320 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:15.325 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.321+0000 7fa724c15640 1 -- 172.21.15.78:0/1839191392 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fa72012f490 msgr2=0x7fa720070610 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:15.325 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.321+0000 7fa724c15640 1 --2- 172.21.15.78:0/1839191392 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fa72012f490 0x7fa720070610 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:15.325 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.321+0000 7fa724c15640 1 -- 172.21.15.78:0/1839191392 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fa720072de0 con 0x7fa72012ed20 2024-09-21T09:22:15.325 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.322+0000 7fa724c15640 1 --2- 172.21.15.78:0/1839191392 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fa72012ed20 0x7fa7201c9860 secure :-1 s=READY pgs=15 cs=0 l=1 rev1=1 crypto rx=0x7fa71c00b3f0 tx=0x7fa71c00b8b0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:15.325 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.322+0000 7fa715ffb640 1 -- 172.21.15.78:0/1839191392 <== mon.2 v2:172.21.15.78:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa71c015020 con 0x7fa72012ed20 2024-09-21T09:22:15.325 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.322+0000 7fa715ffb640 1 -- 172.21.15.78:0/1839191392 <== mon.2 v2:172.21.15.78:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fa71c011920 con 0x7fa72012ed20 2024-09-21T09:22:15.325 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.322+0000 7fa72767a640 1 -- 172.21.15.78:0/1839191392 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fa720073080 con 0x7fa72012ed20 2024-09-21T09:22:15.325 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.323+0000 7fa72767a640 1 -- 172.21.15.78:0/1839191392 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7fa720072680 con 0x7fa72012ed20 2024-09-21T09:22:15.326 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.323+0000 7fa715ffb640 1 -- 172.21.15.78:0/1839191392 <== mon.2 v2:172.21.15.78:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa71c015020 con 0x7fa72012ed20 2024-09-21T09:22:15.327 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.325+0000 7fa715ffb640 1 -- 172.21.15.78:0/1839191392 <== mon.2 v2:172.21.15.78:3302/0 4 ==== mgrmap(e 6) ==== 55351+0+0 (secure 0 0 0) 0x7fa71c018020 con 0x7fa72012ed20 2024-09-21T09:22:15.327 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.325+0000 7fa72767a640 1 -- 172.21.15.78:0/1839191392 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fa720099420 con 0x7fa72012ed20 2024-09-21T09:22:15.328 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.325+0000 7fa725416640 1 -- 172.21.15.78:0/1839191392 >> v1:172.21.15.78:6816/2602406893 conn(0x7fa704047a20 legacy=0x7fa704049ee0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.78:6816/2602406893 2024-09-21T09:22:15.328 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.326+0000 7fa715ffb640 1 -- 172.21.15.78:0/1839191392 <== mon.2 v2:172.21.15.78:3302/0 5 ==== osd_map(19..19 src has 1..19) ==== 2812+0+0 (secure 0 0 0) 0x7fa71c011630 con 0x7fa72012ed20 2024-09-21T09:22:15.335 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.333+0000 7fa715ffb640 1 -- 172.21.15.78:0/1839191392 <== mon.2 v2:172.21.15.78:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+148921 (secure 0 0 0) 0x7fa720099420 con 0x7fa72012ed20 2024-09-21T09:22:15.437 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.435+0000 7eff477fe640 1 -- 172.21.15.78:0/3868308737 >> v1:172.21.15.78:6816/2602406893 conn(0x7eff340477f0 legacy=0x7eff34049cb0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.78:6816/2602406893 2024-09-21T09:22:15.465 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.463+0000 7fab9e40c640 1 -- 172.21.15.78:0/1688357588 >> v1:172.21.15.78:6816/2602406893 conn(0x7fab7c0475f0 legacy=0x7fab7c049ab0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.78:6816/2602406893 2024-09-21T09:22:15.515 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.513+0000 7f927a7fc640 1 -- 172.21.15.78:0/2510468278 >> v1:172.21.15.78:6816/2602406893 conn(0x7f92680476c0 legacy=0x7f9268049b80 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.78:6816/2602406893 2024-09-21T09:22:15.518 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.516+0000 7eff4e108640 1 -- 172.21.15.78:0/3868308737 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 0} v 0) -- 0x7eff480699c0 con 0x7eff48094c90 2024-09-21T09:22:15.519 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.517+0000 7eff2ffff640 1 -- 172.21.15.78:0/3868308737 <== mon.0 v2:172.21.15.78:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 0}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7eff3c03cc60 con 0x7eff48094c90 2024-09-21T09:22:15.519 INFO:teuthology.orchestra.run.smithi078.stdout:38654705682 2024-09-21T09:22:15.522 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.519+0000 7eff4e108640 1 -- 172.21.15.78:0/3868308737 >> v1:172.21.15.78:6816/2602406893 conn(0x7eff340477f0 legacy=0x7eff34049cb0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-21T09:22:15.522 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.519+0000 7eff4e108640 1 -- 172.21.15.78:0/3868308737 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7eff48094c90 msgr2=0x7eff4812db80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:15.522 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.519+0000 7eff4e108640 1 --2- 172.21.15.78:0/3868308737 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7eff48094c90 0x7eff4812db80 secure :-1 s=READY pgs=34 cs=0 l=1 rev1=1 crypto rx=0x7eff3c00ed30 tx=0x7eff3c00c6a0 comp rx=0 tx=0).stop 2024-09-21T09:22:15.522 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.519+0000 7eff4e108640 1 -- 172.21.15.78:0/3868308737 shutdown_connections 2024-09-21T09:22:15.522 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.519+0000 7eff4e108640 1 --2- 172.21.15.78:0/3868308737 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7eff480961b0 0x7eff4812e0c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:15.522 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.519+0000 7eff4e108640 1 --2- 172.21.15.78:0/3868308737 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7eff48094c90 0x7eff4812db80 unknown :-1 s=CLOSED pgs=34 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:15.522 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.519+0000 7eff4e108640 1 --2- 172.21.15.78:0/3868308737 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7eff48094370 0x7eff4812d640 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:15.522 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.519+0000 7eff4e108640 1 -- 172.21.15.78:0/3868308737 >> 172.21.15.78:0/3868308737 conn(0x7eff4809e610 msgr2=0x7eff48128b80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:15.522 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.520+0000 7eff4e108640 1 -- 172.21.15.78:0/3868308737 shutdown_connections 2024-09-21T09:22:15.523 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.520+0000 7eff4e108640 1 -- 172.21.15.78:0/3868308737 wait complete. 2024-09-21T09:22:15.529 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.527+0000 7fa725416640 1 -- 172.21.15.78:0/1839191392 >> v1:172.21.15.78:6816/2602406893 conn(0x7fa704047a20 legacy=0x7fa704049ee0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.78:6816/2602406893 2024-09-21T09:22:15.538 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705683 got 38654705682 for osd.0 2024-09-21T09:22:15.579 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.577+0000 7fab9f40e640 1 -- 172.21.15.78:0/1688357588 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 2} v 0) -- 0x7fab80005850 con 0x7fab900afb00 2024-09-21T09:22:15.579 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.577+0000 7fab8effd640 1 -- 172.21.15.78:0/1688357588 <== mon.0 v2:172.21.15.78:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 2}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7fab8807c6f0 con 0x7fab900afb00 2024-09-21T09:22:15.580 INFO:teuthology.orchestra.run.smithi078.stdout:38654705684 2024-09-21T09:22:15.582 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.580+0000 7fab9f40e640 1 -- 172.21.15.78:0/1688357588 >> v1:172.21.15.78:6816/2602406893 conn(0x7fab7c0475f0 legacy=0x7fab7c049ab0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-21T09:22:15.583 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.580+0000 7fab9f40e640 1 -- 172.21.15.78:0/1688357588 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fab900afb00 msgr2=0x7fab90013160 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:15.583 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.580+0000 7fab9f40e640 1 --2- 172.21.15.78:0/1688357588 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fab900afb00 0x7fab90013160 secure :-1 s=READY pgs=37 cs=0 l=1 rev1=1 crypto rx=0x7fab880098a0 tx=0x7fab88004a50 comp rx=0 tx=0).stop 2024-09-21T09:22:15.583 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.580+0000 7fab9f40e640 1 -- 172.21.15.78:0/1688357588 shutdown_connections 2024-09-21T09:22:15.583 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.580+0000 7fab9f40e640 1 --2- 172.21.15.78:0/1688357588 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fab900136a0 0x7fab900aa620 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:15.583 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.580+0000 7fab9f40e640 1 --2- 172.21.15.78:0/1688357588 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fab900afb00 0x7fab90013160 unknown :-1 s=CLOSED pgs=37 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:15.583 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.580+0000 7fab9f40e640 1 --2- 172.21.15.78:0/1688357588 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fab900186d0 0x7fab90012c20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:15.583 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.580+0000 7fab9f40e640 1 -- 172.21.15.78:0/1688357588 >> 172.21.15.78:0/1688357588 conn(0x7fab90005c50 msgr2=0x7fab9001b5b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:15.583 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.580+0000 7fab9f40e640 1 -- 172.21.15.78:0/1688357588 shutdown_connections 2024-09-21T09:22:15.583 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.580+0000 7fab9f40e640 1 -- 172.21.15.78:0/1688357588 wait complete. 2024-09-21T09:22:15.598 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705685 got 38654705684 for osd.2 2024-09-21T09:22:15.648 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.646+0000 7f9281627640 1 -- 172.21.15.78:0/2510468278 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 3} v 0) -- 0x7f927c0697f0 con 0x7f927c070640 2024-09-21T09:22:15.649 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.646+0000 7f92637fe640 1 -- 172.21.15.78:0/2510468278 <== mon.0 v2:172.21.15.78:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 3}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7f926c07c460 con 0x7f927c070640 2024-09-21T09:22:15.649 INFO:teuthology.orchestra.run.smithi078.stdout:38654705682 2024-09-21T09:22:15.651 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.649+0000 7f9281627640 1 -- 172.21.15.78:0/2510468278 >> v1:172.21.15.78:6816/2602406893 conn(0x7f92680476c0 legacy=0x7f9268049b80 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-21T09:22:15.651 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.649+0000 7f9281627640 1 -- 172.21.15.78:0/2510468278 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f927c070640 msgr2=0x7f927c1b2190 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:15.651 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.649+0000 7f9281627640 1 --2- 172.21.15.78:0/2510468278 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f927c070640 0x7f927c1b2190 secure :-1 s=READY pgs=40 cs=0 l=1 rev1=1 crypto rx=0x7f926c006010 tx=0x7f926c003820 comp rx=0 tx=0).stop 2024-09-21T09:22:15.651 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.649+0000 7fa72767a640 1 -- 172.21.15.78:0/1839191392 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 1} v 0) -- 0x7fa7200738c0 con 0x7fa72012ed20 2024-09-21T09:22:15.652 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.649+0000 7f9281627640 1 -- 172.21.15.78:0/2510468278 shutdown_connections 2024-09-21T09:22:15.652 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.649+0000 7f9281627640 1 --2- 172.21.15.78:0/2510468278 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f927c097740 0x7f927c1b2c10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:15.652 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.649+0000 7f9281627640 1 --2- 172.21.15.78:0/2510468278 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f927c070f60 0x7f927c1b26d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:15.652 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.649+0000 7f9281627640 1 --2- 172.21.15.78:0/2510468278 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f927c070640 0x7f927c1b2190 unknown :-1 s=CLOSED pgs=40 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:15.652 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.649+0000 7f9281627640 1 -- 172.21.15.78:0/2510468278 >> 172.21.15.78:0/2510468278 conn(0x7f927c09dde0 msgr2=0x7f927c098d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:15.652 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.649+0000 7fa715ffb640 1 -- 172.21.15.78:0/1839191392 <== mon.2 v2:172.21.15.78:3302/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 1}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7fa71c01f020 con 0x7fa72012ed20 2024-09-21T09:22:15.652 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.649+0000 7f9281627640 1 -- 172.21.15.78:0/2510468278 shutdown_connections 2024-09-21T09:22:15.653 INFO:teuthology.orchestra.run.smithi078.stdout:38654705682 2024-09-21T09:22:15.653 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.649+0000 7f9281627640 1 -- 172.21.15.78:0/2510468278 wait complete. 2024-09-21T09:22:15.654 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.651+0000 7fa72767a640 1 -- 172.21.15.78:0/1839191392 >> v1:172.21.15.78:6816/2602406893 conn(0x7fa704047a20 legacy=0x7fa704049ee0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-21T09:22:15.654 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.651+0000 7fa72767a640 1 -- 172.21.15.78:0/1839191392 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fa72012ed20 msgr2=0x7fa7201c9860 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:15.654 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.651+0000 7fa72767a640 1 --2- 172.21.15.78:0/1839191392 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fa72012ed20 0x7fa7201c9860 secure :-1 s=READY pgs=15 cs=0 l=1 rev1=1 crypto rx=0x7fa71c00b3f0 tx=0x7fa71c00b8b0 comp rx=0 tx=0).stop 2024-09-21T09:22:15.654 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.651+0000 7fa72767a640 1 -- 172.21.15.78:0/1839191392 shutdown_connections 2024-09-21T09:22:15.654 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.651+0000 7fa72767a640 1 --2- 172.21.15.78:0/1839191392 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fa72012f490 0x7fa720070610 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:15.654 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.651+0000 7fa72767a640 1 --2- 172.21.15.78:0/1839191392 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fa72012ed20 0x7fa7201c9860 unknown :-1 s=CLOSED pgs=15 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:15.654 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.651+0000 7fa72767a640 1 --2- 172.21.15.78:0/1839191392 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fa720073fd0 0x7fa7201c9320 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:15.654 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.651+0000 7fa72767a640 1 -- 172.21.15.78:0/1839191392 >> 172.21.15.78:0/1839191392 conn(0x7fa72009e610 msgr2=0x7fa720096670 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:15.654 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.651+0000 7fa72767a640 1 -- 172.21.15.78:0/1839191392 shutdown_connections 2024-09-21T09:22:15.654 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:15.651+0000 7fa72767a640 1 -- 172.21.15.78:0/1839191392 wait complete. 2024-09-21T09:22:15.665 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705683 got 38654705682 for osd.1 2024-09-21T09:22:15.668 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705683 got 38654705682 for osd.3 2024-09-21T09:22:16.539 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.0 2024-09-21T09:22:16.599 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.2 2024-09-21T09:22:16.643 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.640+0000 7f614e574640 1 -- 172.21.15.78:0/2453679871 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f6148098d40 msgr2=0x7f6148099120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:16.643 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.640+0000 7f614e574640 1 --2- 172.21.15.78:0/2453679871 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f6148098d40 0x7f6148099120 secure :-1 s=READY pgs=23 cs=0 l=1 rev1=1 crypto rx=0x7f6138009900 tx=0x7f613802a4a0 comp rx=0 tx=0).stop 2024-09-21T09:22:16.643 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.640+0000 7f614e574640 1 -- 172.21.15.78:0/2453679871 shutdown_connections 2024-09-21T09:22:16.643 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.640+0000 7f614e574640 1 --2- 172.21.15.78:0/2453679871 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f614812f1c0 0x7f614812f650 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:16.643 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.640+0000 7f614e574640 1 --2- 172.21.15.78:0/2453679871 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f6148098d40 0x7f6148099120 unknown :-1 s=CLOSED pgs=23 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:16.643 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.640+0000 7f614e574640 1 -- 172.21.15.78:0/2453679871 >> 172.21.15.78:0/2453679871 conn(0x7f61480a1790 msgr2=0x7f61480a1ba0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:16.643 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.640+0000 7f614e574640 1 -- 172.21.15.78:0/2453679871 shutdown_connections 2024-09-21T09:22:16.643 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.640+0000 7f614e574640 1 -- 172.21.15.78:0/2453679871 wait complete. 2024-09-21T09:22:16.645 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.641+0000 7f614e574640 1 Processor -- start 2024-09-21T09:22:16.645 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.641+0000 7f614e574640 1 -- start start 2024-09-21T09:22:16.645 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.641+0000 7f614e574640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f6148098d40 0x7f6148071520 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:16.645 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.641+0000 7f614e574640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f6148099660 0x7f6148071a60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:16.645 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.641+0000 7f614e574640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f614812f1c0 0x7f6148071fa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:16.645 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.641+0000 7f614e574640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7f6148137910 con 0x7f6148098d40 2024-09-21T09:22:16.645 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.641+0000 7f614e574640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7f6148137790 con 0x7f614812f1c0 2024-09-21T09:22:16.645 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.641+0000 7f614e574640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7f6148137a90 con 0x7f6148099660 2024-09-21T09:22:16.645 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.642+0000 7f61477fe640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f6148099660 0x7f6148071a60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:16.646 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.642+0000 7f614cb11640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f614812f1c0 0x7f6148071fa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:16.646 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.642+0000 7f6147fff640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f6148098d40 0x7f6148071520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:16.646 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.642+0000 7f61477fe640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f6148099660 0x7f6148071a60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3302/0 says I am v2:172.21.15.78:53786/0 (socket says 172.21.15.78:53786) 2024-09-21T09:22:16.646 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.642+0000 7f61477fe640 1 -- 172.21.15.78:0/2952809743 learned_addr learned my addr 172.21.15.78:0/2952809743 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:16.646 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.642+0000 7f614cb11640 1 -- 172.21.15.78:0/2952809743 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f6148099660 msgr2=0x7f6148071a60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:16.646 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.642+0000 7f614cb11640 1 --2- 172.21.15.78:0/2952809743 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f6148099660 0x7f6148071a60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:16.646 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.642+0000 7f614cb11640 1 -- 172.21.15.78:0/2952809743 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f6148098d40 msgr2=0x7f6148071520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:16.646 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.642+0000 7f614cb11640 1 --2- 172.21.15.78:0/2952809743 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f6148098d40 0x7f6148071520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:16.646 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.642+0000 7f614cb11640 1 -- 172.21.15.78:0/2952809743 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f6148129460 con 0x7f614812f1c0 2024-09-21T09:22:16.646 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.642+0000 7f6147fff640 1 --2- 172.21.15.78:0/2952809743 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f6148098d40 0x7f6148071520 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:22:16.646 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.642+0000 7f614cb11640 1 --2- 172.21.15.78:0/2952809743 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f614812f1c0 0x7f6148071fa0 secure :-1 s=READY pgs=24 cs=0 l=1 rev1=1 crypto rx=0x7f614000ec80 tx=0x7f614000c6a0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:16.646 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.642+0000 7f6144ff9640 1 -- 172.21.15.78:0/2952809743 <== mon.1 v2:172.21.15.78:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6140003b70 con 0x7f614812f1c0 2024-09-21T09:22:16.646 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.642+0000 7f6144ff9640 1 -- 172.21.15.78:0/2952809743 <== mon.1 v2:172.21.15.78:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f614000ee90 con 0x7f614812f1c0 2024-09-21T09:22:16.647 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.642+0000 7f614e574640 1 -- 172.21.15.78:0/2952809743 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f61480724e0 con 0x7f614812f1c0 2024-09-21T09:22:16.647 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.642+0000 7f6144ff9640 1 -- 172.21.15.78:0/2952809743 <== mon.1 v2:172.21.15.78:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6140004020 con 0x7f614812f1c0 2024-09-21T09:22:16.647 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.642+0000 7f614e574640 1 -- 172.21.15.78:0/2952809743 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7f61481cad90 con 0x7f614812f1c0 2024-09-21T09:22:16.647 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.643+0000 7f6144ff9640 1 -- 172.21.15.78:0/2952809743 <== mon.1 v2:172.21.15.78:3301/0 4 ==== mgrmap(e 7) ==== 63244+0+0 (secure 0 0 0) 0x7f61400041c0 con 0x7f614812f1c0 2024-09-21T09:22:16.647 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.644+0000 7f6144ff9640 1 -- 172.21.15.78:0/2952809743 <== mon.1 v2:172.21.15.78:3301/0 5 ==== osd_map(20..20 src has 1..20) ==== 3027+0+0 (secure 0 0 0) 0x7f6140069780 con 0x7f614812f1c0 2024-09-21T09:22:16.647 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.644+0000 7f614e574640 1 -- 172.21.15.78:0/2952809743 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f614812dfa0 con 0x7f614812f1c0 2024-09-21T09:22:16.649 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.647+0000 7f6144ff9640 1 -- 172.21.15.78:0/2952809743 <== mon.1 v2:172.21.15.78:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+148921 (secure 0 0 0) 0x7f614812dfa0 con 0x7f614812f1c0 2024-09-21T09:22:16.666 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.1 2024-09-21T09:22:16.668 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.3 2024-09-21T09:22:16.704 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.701+0000 7f2e6776a640 1 -- 172.21.15.78:0/4235632106 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f2e60098d40 msgr2=0x7f2e60099120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:16.704 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.701+0000 7f2e6776a640 1 --2- 172.21.15.78:0/4235632106 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f2e60098d40 0x7f2e60099120 secure :-1 s=READY pgs=25 cs=0 l=1 rev1=1 crypto rx=0x7f2e5c009830 tx=0x7f2e5c02a360 comp rx=0 tx=0).stop 2024-09-21T09:22:16.704 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.701+0000 7f2e6776a640 1 -- 172.21.15.78:0/4235632106 shutdown_connections 2024-09-21T09:22:16.704 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.701+0000 7f2e6776a640 1 --2- 172.21.15.78:0/4235632106 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f2e60099660 0x7f2e6012eab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:16.704 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.701+0000 7f2e6776a640 1 --2- 172.21.15.78:0/4235632106 >> v2:172.21.15.78:3300/0 conn(0x7f2e6012eff0 0x7f2e6012f480 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:16.704 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.701+0000 7f2e6776a640 1 --2- 172.21.15.78:0/4235632106 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f2e60098d40 0x7f2e60099120 unknown :-1 s=CLOSED pgs=25 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:16.704 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.701+0000 7f2e6776a640 1 -- 172.21.15.78:0/4235632106 >> 172.21.15.78:0/4235632106 conn(0x7f2e600a1790 msgr2=0x7f2e600a1ba0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:16.704 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.701+0000 7f2e6776a640 1 -- 172.21.15.78:0/4235632106 shutdown_connections 2024-09-21T09:22:16.704 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.701+0000 7f2e6776a640 1 -- 172.21.15.78:0/4235632106 wait complete. 2024-09-21T09:22:16.705 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.702+0000 7f2e6776a640 1 Processor -- start 2024-09-21T09:22:16.706 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.702+0000 7f2e6776a640 1 -- start start 2024-09-21T09:22:16.706 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.702+0000 7f2e6776a640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f2e60098d40 0x7f2e601c4a60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:16.706 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.702+0000 7f2e6776a640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f2e60099660 0x7f2e601c4fa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:16.706 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.702+0000 7f2e6776a640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f2e6012eff0 0x7f2e60132a40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:16.706 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.702+0000 7f2e6776a640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7f2e60136570 con 0x7f2e60099660 2024-09-21T09:22:16.706 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.702+0000 7f2e6776a640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7f2e601363f0 con 0x7f2e60098d40 2024-09-21T09:22:16.706 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.702+0000 7f2e6776a640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7f2e601366f0 con 0x7f2e6012eff0 2024-09-21T09:22:16.707 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.703+0000 7f2e64d05640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f2e60099660 0x7f2e601c4fa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:16.707 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.703+0000 7f2e65d07640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f2e6012eff0 0x7f2e60132a40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:16.707 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.703+0000 7f2e64d05640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f2e60099660 0x7f2e601c4fa0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3300/0 says I am v2:172.21.15.78:51260/0 (socket says 172.21.15.78:51260) 2024-09-21T09:22:16.707 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.703+0000 7f2e65d07640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f2e6012eff0 0x7f2e60132a40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3302/0 says I am v2:172.21.15.78:53802/0 (socket says 172.21.15.78:53802) 2024-09-21T09:22:16.707 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.703+0000 7f2e64d05640 1 -- 172.21.15.78:0/3907727252 learned_addr learned my addr 172.21.15.78:0/3907727252 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:16.707 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.703+0000 7f2e65506640 1 --2- 172.21.15.78:0/3907727252 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f2e60098d40 0x7f2e601c4a60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:16.707 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.703+0000 7f2e64d05640 1 -- 172.21.15.78:0/3907727252 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f2e6012eff0 msgr2=0x7f2e60132a40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:16.707 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.703+0000 7f2e64d05640 1 --2- 172.21.15.78:0/3907727252 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f2e6012eff0 0x7f2e60132a40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:16.707 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.703+0000 7f2e64d05640 1 -- 172.21.15.78:0/3907727252 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f2e60098d40 msgr2=0x7f2e601c4a60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:16.707 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.703+0000 7f2e64d05640 1 --2- 172.21.15.78:0/3907727252 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f2e60098d40 0x7f2e601c4a60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:16.707 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.703+0000 7f2e64d05640 1 -- 172.21.15.78:0/3907727252 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f2e601354b0 con 0x7f2e60099660 2024-09-21T09:22:16.707 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.703+0000 7f2e64d05640 1 --2- 172.21.15.78:0/3907727252 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f2e60099660 0x7f2e601c4fa0 secure :-1 s=READY pgs=46 cs=0 l=1 rev1=1 crypto rx=0x7f2e5000e9f0 tx=0x7f2e5000eeb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:16.707 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.703+0000 7f2e55ffb640 1 -- 172.21.15.78:0/3907727252 <== mon.0 v2:172.21.15.78:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2e5000cc60 con 0x7f2e60099660 2024-09-21T09:22:16.708 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.703+0000 7f2e6776a640 1 -- 172.21.15.78:0/3907727252 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f2e60134cb0 con 0x7f2e60099660 2024-09-21T09:22:16.708 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.703+0000 7f2e55ffb640 1 -- 172.21.15.78:0/3907727252 <== mon.0 v2:172.21.15.78:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f2e5000ce00 con 0x7f2e60099660 2024-09-21T09:22:16.708 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.703+0000 7f2e55ffb640 1 -- 172.21.15.78:0/3907727252 <== mon.0 v2:172.21.15.78:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2e50005240 con 0x7f2e60099660 2024-09-21T09:22:16.708 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.703+0000 7f2e6776a640 1 -- 172.21.15.78:0/3907727252 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f2e601c56c0 con 0x7f2e60099660 2024-09-21T09:22:16.708 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.705+0000 7f2e55ffb640 1 -- 172.21.15.78:0/3907727252 <== mon.0 v2:172.21.15.78:3300/0 4 ==== mgrmap(e 7) ==== 63244+0+0 (secure 0 0 0) 0x7f2e500054a0 con 0x7f2e60099660 2024-09-21T09:22:16.708 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.705+0000 7f2e6776a640 1 -- 172.21.15.78:0/3907727252 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f2e60099120 con 0x7f2e60099660 2024-09-21T09:22:16.708 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.705+0000 7f2e55ffb640 1 -- 172.21.15.78:0/3907727252 <== mon.0 v2:172.21.15.78:3300/0 5 ==== osd_map(20..20 src has 1..20) ==== 3027+0+0 (secure 0 0 0) 0x7f2e50069610 con 0x7f2e60099660 2024-09-21T09:22:16.714 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.712+0000 7f2e55ffb640 1 -- 172.21.15.78:0/3907727252 <== mon.0 v2:172.21.15.78:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+148921 (secure 0 0 0) 0x7f2e601354b0 con 0x7f2e60099660 2024-09-21T09:22:16.774 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.771+0000 7fa08b89a640 1 -- 172.21.15.78:0/3780445024 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fa08412cf30 msgr2=0x7fa08412d380 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:16.774 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.771+0000 7fa08b89a640 1 --2- 172.21.15.78:0/3780445024 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fa08412cf30 0x7fa08412d380 secure :-1 s=READY pgs=48 cs=0 l=1 rev1=1 crypto rx=0x7fa07c005fd0 tx=0x7fa07c02a980 comp rx=0 tx=0).stop 2024-09-21T09:22:16.774 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.771+0000 7fa08b89a640 1 -- 172.21.15.78:0/3780445024 shutdown_connections 2024-09-21T09:22:16.774 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.771+0000 7fa08b89a640 1 --2- 172.21.15.78:0/3780445024 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fa08412cf30 0x7fa08412d380 unknown :-1 s=CLOSED pgs=48 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:16.774 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.771+0000 7fa08b89a640 1 --2- 172.21.15.78:0/3780445024 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fa084129510 0x7fa0841298f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:16.774 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.771+0000 7fa08b89a640 1 -- 172.21.15.78:0/3780445024 >> 172.21.15.78:0/3780445024 conn(0x7fa08409d370 msgr2=0x7fa08409d780 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:16.774 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.771+0000 7fa08b89a640 1 -- 172.21.15.78:0/3780445024 shutdown_connections 2024-09-21T09:22:16.774 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.771+0000 7fa08b89a640 1 -- 172.21.15.78:0/3780445024 wait complete. 2024-09-21T09:22:16.775 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.773+0000 7fa08b89a640 1 Processor -- start 2024-09-21T09:22:16.775 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.773+0000 7fa08b89a640 1 -- start start 2024-09-21T09:22:16.776 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.773+0000 7fa08b89a640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fa084129510 0x7fa084123090 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:16.776 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.773+0000 7fa08b89a640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fa084129e30 0x7fa08406f410 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:16.776 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.773+0000 7fa08b89a640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fa08412cf30 0x7fa08406f950 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:16.776 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.773+0000 7fa08b89a640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7fa084135750 con 0x7fa08412cf30 2024-09-21T09:22:16.777 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.773+0000 7fa089636640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fa084129510 0x7fa084123090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:16.777 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.773+0000 7fa08b89a640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7fa0841355d0 con 0x7fa084129510 2024-09-21T09:22:16.777 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.773+0000 7fa08b89a640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7fa0841358d0 con 0x7fa084129e30 2024-09-21T09:22:16.777 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.773+0000 7fa089636640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fa084129510 0x7fa084123090 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3301/0 says I am v2:172.21.15.78:60798/0 (socket says 172.21.15.78:60798) 2024-09-21T09:22:16.777 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.773+0000 7fa089e37640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fa08412cf30 0x7fa08406f950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:16.777 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.773+0000 7fa089636640 1 -- 172.21.15.78:0/3911975638 learned_addr learned my addr 172.21.15.78:0/3911975638 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:16.777 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.773+0000 7fa088e35640 1 --2- 172.21.15.78:0/3911975638 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fa084129e30 0x7fa08406f410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:16.777 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.774+0000 7fa089e37640 1 -- 172.21.15.78:0/3911975638 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fa084129e30 msgr2=0x7fa08406f410 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:16.778 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.774+0000 7fa089e37640 1 --2- 172.21.15.78:0/3911975638 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fa084129e30 0x7fa08406f410 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:16.778 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.774+0000 7fa089e37640 1 -- 172.21.15.78:0/3911975638 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fa084129510 msgr2=0x7fa084123090 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:16.778 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.774+0000 7fa089e37640 1 --2- 172.21.15.78:0/3911975638 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fa084129510 0x7fa084123090 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:16.778 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.774+0000 7fa089e37640 1 -- 172.21.15.78:0/3911975638 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fa084126f70 con 0x7fa08412cf30 2024-09-21T09:22:16.778 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.774+0000 7fa089636640 1 --2- 172.21.15.78:0/3911975638 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fa084129510 0x7fa084123090 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:22:16.778 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.774+0000 7fa089e37640 1 --2- 172.21.15.78:0/3911975638 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fa08412cf30 0x7fa08406f950 secure :-1 s=READY pgs=49 cs=0 l=1 rev1=1 crypto rx=0x7fa07c032000 tx=0x7fa07c0093d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:16.778 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.774+0000 7fa079ffb640 1 -- 172.21.15.78:0/3911975638 <== mon.0 v2:172.21.15.78:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa07c034070 con 0x7fa08412cf30 2024-09-21T09:22:16.778 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.774+0000 7fa079ffb640 1 -- 172.21.15.78:0/3911975638 <== mon.0 v2:172.21.15.78:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fa07c043a50 con 0x7fa08412cf30 2024-09-21T09:22:16.778 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.774+0000 7fa08b89a640 1 -- 172.21.15.78:0/3911975638 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fa0841313f0 con 0x7fa08412cf30 2024-09-21T09:22:16.778 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.774+0000 7fa079ffb640 1 -- 172.21.15.78:0/3911975638 <== mon.0 v2:172.21.15.78:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa07c004040 con 0x7fa08412cf30 2024-09-21T09:22:16.778 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.775+0000 7fa08b89a640 1 -- 172.21.15.78:0/3911975638 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fa0841235d0 con 0x7fa08412cf30 2024-09-21T09:22:16.779 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.776+0000 7fa079ffb640 1 -- 172.21.15.78:0/3911975638 <== mon.0 v2:172.21.15.78:3300/0 4 ==== mgrmap(e 7) ==== 63244+0+0 (secure 0 0 0) 0x7fa07c037020 con 0x7fa08412cf30 2024-09-21T09:22:16.779 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.776+0000 7fa08b89a640 1 -- 172.21.15.78:0/3911975638 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fa0841298f0 con 0x7fa08412cf30 2024-09-21T09:22:16.779 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.776+0000 7fa079ffb640 1 -- 172.21.15.78:0/3911975638 <== mon.0 v2:172.21.15.78:3300/0 5 ==== osd_map(20..20 src has 1..20) ==== 3027+0+0 (secure 0 0 0) 0x7fa07c092640 con 0x7fa08412cf30 2024-09-21T09:22:16.782 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.780+0000 7fa079ffb640 1 -- 172.21.15.78:0/3911975638 <== mon.0 v2:172.21.15.78:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+148921 (secure 0 0 0) 0x7fa07c091490 con 0x7fa08412cf30 2024-09-21T09:22:16.810 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.808+0000 7f3ae359e640 1 -- 172.21.15.78:0/4033832606 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f3ae409a050 msgr2=0x7f3ae4097400 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:16.810 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.808+0000 7f3ae359e640 1 --2- 172.21.15.78:0/4033832606 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f3ae409a050 0x7f3ae4097400 secure :-1 s=READY pgs=18 cs=0 l=1 rev1=1 crypto rx=0x7f3ad8009830 tx=0x7f3ad802a3f0 comp rx=0 tx=0).stop 2024-09-21T09:22:16.810 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.808+0000 7f3ae359e640 1 -- 172.21.15.78:0/4033832606 shutdown_connections 2024-09-21T09:22:16.811 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.808+0000 7f3ae359e640 1 --2- 172.21.15.78:0/4033832606 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f3ae409a050 0x7f3ae4097400 unknown :-1 s=CLOSED pgs=18 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:16.811 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.808+0000 7f3ae359e640 1 --2- 172.21.15.78:0/4033832606 >> v2:172.21.15.78:3300/0 conn(0x7f3ae4099730 0x7f3ae4099b10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:16.811 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.808+0000 7f3ae359e640 1 -- 172.21.15.78:0/4033832606 >> 172.21.15.78:0/4033832606 conn(0x7f3ae40a1850 msgr2=0x7f3ae409dd20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:16.811 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.808+0000 7f3ae359e640 1 -- 172.21.15.78:0/4033832606 shutdown_connections 2024-09-21T09:22:16.811 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.808+0000 7f3ae359e640 1 -- 172.21.15.78:0/4033832606 wait complete. 2024-09-21T09:22:16.812 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.809+0000 7f3ae359e640 1 Processor -- start 2024-09-21T09:22:16.812 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.809+0000 7f3ae359e640 1 -- start start 2024-09-21T09:22:16.812 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.809+0000 7f3ae359e640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f3ae4097940 0x7f3ae41c4bc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:16.812 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.809+0000 7f3ae359e640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f3ae4099730 0x7f3ae41c5100 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:16.812 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.809+0000 7f3ae359e640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f3ae409a050 0x7f3ae41c5640 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:16.812 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.810+0000 7f3ae259c640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f3ae4097940 0x7f3ae41c4bc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:16.812 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.810+0000 7f3ae2d9d640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f3ae409a050 0x7f3ae41c5640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:16.812 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.810+0000 7f3ae1d9b640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f3ae4099730 0x7f3ae41c5100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:16.813 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.810+0000 7f3ae2d9d640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f3ae409a050 0x7f3ae41c5640 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3300/0 says I am v2:172.21.15.78:51296/0 (socket says 172.21.15.78:51296) 2024-09-21T09:22:16.813 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.810+0000 7f3ae2d9d640 1 -- 172.21.15.78:0/3330064920 learned_addr learned my addr 172.21.15.78:0/3330064920 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:16.813 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.810+0000 7f3ae359e640 1 -- 172.21.15.78:0/3330064920 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7f3ae4137890 con 0x7f3ae409a050 2024-09-21T09:22:16.813 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.810+0000 7f3ae359e640 1 -- 172.21.15.78:0/3330064920 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7f3ae4137710 con 0x7f3ae4099730 2024-09-21T09:22:16.813 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.810+0000 7f3ae359e640 1 -- 172.21.15.78:0/3330064920 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7f3ae4137a10 con 0x7f3ae4097940 2024-09-21T09:22:16.813 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.810+0000 7f3ae2d9d640 1 -- 172.21.15.78:0/3330064920 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f3ae4097940 msgr2=0x7f3ae41c4bc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:16.814 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.810+0000 7f3ae2d9d640 1 --2- 172.21.15.78:0/3330064920 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f3ae4097940 0x7f3ae41c4bc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:16.814 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.810+0000 7f3ae2d9d640 1 -- 172.21.15.78:0/3330064920 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f3ae4099730 msgr2=0x7f3ae41c5100 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:16.814 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.810+0000 7f3ae2d9d640 1 --2- 172.21.15.78:0/3330064920 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f3ae4099730 0x7f3ae41c5100 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:16.815 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.810+0000 7f3ae2d9d640 1 -- 172.21.15.78:0/3330064920 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f3ae4129310 con 0x7f3ae409a050 2024-09-21T09:22:16.815 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.810+0000 7f3ae1d9b640 1 --2- 172.21.15.78:0/3330064920 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f3ae4099730 0x7f3ae41c5100 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:22:16.815 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.810+0000 7f3ae259c640 1 --2- 172.21.15.78:0/3330064920 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f3ae4097940 0x7f3ae41c4bc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:22:16.815 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.810+0000 7f3ae2d9d640 1 --2- 172.21.15.78:0/3330064920 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f3ae409a050 0x7f3ae41c5640 secure :-1 s=READY pgs=51 cs=0 l=1 rev1=1 crypto rx=0x7f3adc00cbc0 tx=0x7f3adc00a430 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:16.815 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.811+0000 7f3ad2ffd640 1 -- 172.21.15.78:0/3330064920 <== mon.0 v2:172.21.15.78:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3adc016020 con 0x7f3ae409a050 2024-09-21T09:22:16.815 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.811+0000 7f3ad2ffd640 1 -- 172.21.15.78:0/3330064920 <== mon.0 v2:172.21.15.78:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f3adc00add0 con 0x7f3ae409a050 2024-09-21T09:22:16.815 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.811+0000 7f3ad2ffd640 1 -- 172.21.15.78:0/3330064920 <== mon.0 v2:172.21.15.78:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3adc012680 con 0x7f3ae409a050 2024-09-21T09:22:16.815 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.811+0000 7f3ae359e640 1 -- 172.21.15.78:0/3330064920 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f3ae4073490 con 0x7f3ae409a050 2024-09-21T09:22:16.815 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.811+0000 7f3ae359e640 1 -- 172.21.15.78:0/3330064920 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f3ae4070cf0 con 0x7f3ae409a050 2024-09-21T09:22:16.815 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.812+0000 7f3ad2ffd640 1 -- 172.21.15.78:0/3330064920 <== mon.0 v2:172.21.15.78:3300/0 4 ==== mgrmap(e 7) ==== 63244+0+0 (secure 0 0 0) 0x7f3adc019020 con 0x7f3ae409a050 2024-09-21T09:22:16.815 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.812+0000 7f3ae359e640 1 -- 172.21.15.78:0/3330064920 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f3ae4099b10 con 0x7f3ae409a050 2024-09-21T09:22:16.815 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.812+0000 7f3ad2ffd640 1 -- 172.21.15.78:0/3330064920 <== mon.0 v2:172.21.15.78:3300/0 5 ==== osd_map(20..20 src has 1..20) ==== 3027+0+0 (secure 0 0 0) 0x7f3adc068e40 con 0x7f3ae409a050 2024-09-21T09:22:16.818 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.816+0000 7f3ad2ffd640 1 -- 172.21.15.78:0/3330064920 <== mon.0 v2:172.21.15.78:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+148921 (secure 0 0 0) 0x7f3adc020030 con 0x7f3ae409a050 2024-09-21T09:22:16.869 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.867+0000 7f614e574640 1 -- 172.21.15.78:0/2952809743 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 0} v 0) -- 0x7f6148099120 con 0x7f614812f1c0 2024-09-21T09:22:16.869 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.867+0000 7f6144ff9640 1 -- 172.21.15.78:0/2952809743 <== mon.1 v2:172.21.15.78:3301/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 0}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7f61400686c0 con 0x7f614812f1c0 2024-09-21T09:22:16.870 INFO:teuthology.orchestra.run.smithi078.stdout:38654705682 2024-09-21T09:22:16.872 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.870+0000 7f614e574640 1 -- 172.21.15.78:0/2952809743 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f614812f1c0 msgr2=0x7f6148071fa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:16.873 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.870+0000 7f614e574640 1 --2- 172.21.15.78:0/2952809743 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f614812f1c0 0x7f6148071fa0 secure :-1 s=READY pgs=24 cs=0 l=1 rev1=1 crypto rx=0x7f614000ec80 tx=0x7f614000c6a0 comp rx=0 tx=0).stop 2024-09-21T09:22:16.873 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.870+0000 7f614e574640 1 -- 172.21.15.78:0/2952809743 shutdown_connections 2024-09-21T09:22:16.873 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.870+0000 7f614e574640 1 --2- 172.21.15.78:0/2952809743 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f614812f1c0 0x7f6148071fa0 unknown :-1 s=CLOSED pgs=24 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:16.873 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.870+0000 7f614e574640 1 --2- 172.21.15.78:0/2952809743 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f6148099660 0x7f6148071a60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:16.873 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.870+0000 7f614e574640 1 --2- 172.21.15.78:0/2952809743 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f6148098d40 0x7f6148071520 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:16.873 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.870+0000 7f614e574640 1 -- 172.21.15.78:0/2952809743 >> 172.21.15.78:0/2952809743 conn(0x7f61480a1790 msgr2=0x7f614809d150 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:16.873 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.870+0000 7f614e574640 1 -- 172.21.15.78:0/2952809743 shutdown_connections 2024-09-21T09:22:16.873 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.870+0000 7f614e574640 1 -- 172.21.15.78:0/2952809743 wait complete. 2024-09-21T09:22:16.886 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705683 got 38654705682 for osd.0 2024-09-21T09:22:16.952 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.949+0000 7f2e6776a640 1 -- 172.21.15.78:0/3907727252 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 2} v 0) -- 0x7f2e600698c0 con 0x7f2e60099660 2024-09-21T09:22:16.952 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.950+0000 7f2e55ffb640 1 -- 172.21.15.78:0/3907727252 <== mon.0 v2:172.21.15.78:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 2}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7f2e50068550 con 0x7f2e60099660 2024-09-21T09:22:16.952 INFO:teuthology.orchestra.run.smithi078.stdout:38654705684 2024-09-21T09:22:16.954 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.951+0000 7f2e6776a640 1 -- 172.21.15.78:0/3907727252 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f2e60099660 msgr2=0x7f2e601c4fa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:16.954 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.951+0000 7f2e6776a640 1 --2- 172.21.15.78:0/3907727252 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f2e60099660 0x7f2e601c4fa0 secure :-1 s=READY pgs=46 cs=0 l=1 rev1=1 crypto rx=0x7f2e5000e9f0 tx=0x7f2e5000eeb0 comp rx=0 tx=0).stop 2024-09-21T09:22:16.954 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.951+0000 7f2e6776a640 1 -- 172.21.15.78:0/3907727252 shutdown_connections 2024-09-21T09:22:16.954 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.951+0000 7f2e6776a640 1 --2- 172.21.15.78:0/3907727252 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f2e6012eff0 0x7f2e60132a40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:16.954 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.951+0000 7f2e6776a640 1 --2- 172.21.15.78:0/3907727252 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f2e60099660 0x7f2e601c4fa0 unknown :-1 s=CLOSED pgs=46 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:16.954 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.952+0000 7f2e6776a640 1 --2- 172.21.15.78:0/3907727252 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f2e60098d40 0x7f2e601c4a60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:16.954 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.952+0000 7f2e6776a640 1 -- 172.21.15.78:0/3907727252 >> 172.21.15.78:0/3907727252 conn(0x7f2e600a1790 msgr2=0x7f2e6009c270 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:16.955 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.952+0000 7f2e6776a640 1 -- 172.21.15.78:0/3907727252 shutdown_connections 2024-09-21T09:22:16.955 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.952+0000 7f2e6776a640 1 -- 172.21.15.78:0/3907727252 wait complete. 2024-09-21T09:22:16.964 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705685 got 38654705684 for osd.2 2024-09-21T09:22:16.976 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.974+0000 7fa08b89a640 1 -- 172.21.15.78:0/3911975638 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 1} v 0) -- 0x7fa08409b880 con 0x7fa08412cf30 2024-09-21T09:22:16.977 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.974+0000 7fa079ffb640 1 -- 172.21.15.78:0/3911975638 <== mon.0 v2:172.21.15.78:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 1}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7fa07c06b580 con 0x7fa08412cf30 2024-09-21T09:22:16.977 INFO:teuthology.orchestra.run.smithi078.stdout:38654705682 2024-09-21T09:22:16.979 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.976+0000 7fa08b89a640 1 -- 172.21.15.78:0/3911975638 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fa08412cf30 msgr2=0x7fa08406f950 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:16.979 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.976+0000 7fa08b89a640 1 --2- 172.21.15.78:0/3911975638 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fa08412cf30 0x7fa08406f950 secure :-1 s=READY pgs=49 cs=0 l=1 rev1=1 crypto rx=0x7fa07c032000 tx=0x7fa07c0093d0 comp rx=0 tx=0).stop 2024-09-21T09:22:16.979 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.976+0000 7fa08b89a640 1 -- 172.21.15.78:0/3911975638 shutdown_connections 2024-09-21T09:22:16.979 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.976+0000 7fa08b89a640 1 --2- 172.21.15.78:0/3911975638 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fa08412cf30 0x7fa08406f950 unknown :-1 s=CLOSED pgs=49 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:16.979 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.976+0000 7fa08b89a640 1 --2- 172.21.15.78:0/3911975638 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fa084129e30 0x7fa08406f410 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:16.979 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.976+0000 7fa08b89a640 1 --2- 172.21.15.78:0/3911975638 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fa084129510 0x7fa084123090 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:16.979 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.976+0000 7fa08b89a640 1 -- 172.21.15.78:0/3911975638 >> 172.21.15.78:0/3911975638 conn(0x7fa08409d370 msgr2=0x7fa08412c610 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:16.979 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.976+0000 7fa08b89a640 1 -- 172.21.15.78:0/3911975638 shutdown_connections 2024-09-21T09:22:16.979 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:16.976+0000 7fa08b89a640 1 -- 172.21.15.78:0/3911975638 wait complete. 2024-09-21T09:22:16.989 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705683 got 38654705682 for osd.1 2024-09-21T09:22:17.029 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:17.026+0000 7f3ae359e640 1 -- 172.21.15.78:0/3330064920 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 3} v 0) -- 0x7f3ae40697f0 con 0x7f3ae409a050 2024-09-21T09:22:17.029 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:17.026+0000 7f3ad2ffd640 1 -- 172.21.15.78:0/3330064920 <== mon.0 v2:172.21.15.78:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 3}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7f3adc067d80 con 0x7f3ae409a050 2024-09-21T09:22:17.029 INFO:teuthology.orchestra.run.smithi078.stdout:38654705682 2024-09-21T09:22:17.031 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:17.028+0000 7f3ae359e640 1 -- 172.21.15.78:0/3330064920 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f3ae409a050 msgr2=0x7f3ae41c5640 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:17.031 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:17.028+0000 7f3ae359e640 1 --2- 172.21.15.78:0/3330064920 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f3ae409a050 0x7f3ae41c5640 secure :-1 s=READY pgs=51 cs=0 l=1 rev1=1 crypto rx=0x7f3adc00cbc0 tx=0x7f3adc00a430 comp rx=0 tx=0).stop 2024-09-21T09:22:17.031 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:17.028+0000 7f3ae359e640 1 -- 172.21.15.78:0/3330064920 shutdown_connections 2024-09-21T09:22:17.031 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:17.028+0000 7f3ae359e640 1 --2- 172.21.15.78:0/3330064920 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f3ae409a050 0x7f3ae41c5640 unknown :-1 s=CLOSED pgs=51 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:17.031 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:17.028+0000 7f3ae359e640 1 --2- 172.21.15.78:0/3330064920 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f3ae4099730 0x7f3ae41c5100 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:17.031 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:17.028+0000 7f3ae359e640 1 --2- 172.21.15.78:0/3330064920 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f3ae4097940 0x7f3ae41c4bc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:17.031 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:17.028+0000 7f3ae359e640 1 -- 172.21.15.78:0/3330064920 >> 172.21.15.78:0/3330064920 conn(0x7f3ae40a1850 msgr2=0x7f3ae409c030 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:17.031 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:17.028+0000 7f3ae359e640 1 -- 172.21.15.78:0/3330064920 shutdown_connections 2024-09-21T09:22:17.031 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:17.028+0000 7f3ae359e640 1 -- 172.21.15.78:0/3330064920 wait complete. 2024-09-21T09:22:17.041 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705683 got 38654705682 for osd.3 2024-09-21T09:22:17.401 INFO:tasks.ceph.mgr.x.smithi078.stderr:2024-09-21T09:22:17.399+0000 7fdc967ba640 -1 mgr.server handle_report got status from non-daemon mon.c 2024-09-21T09:22:17.402 INFO:tasks.ceph.mgr.x.smithi078.stderr:2024-09-21T09:22:17.399+0000 7fdc967ba640 -1 mgr.server handle_report got status from non-daemon mon.a 2024-09-21T09:22:17.402 INFO:tasks.ceph.mgr.x.smithi078.stderr:2024-09-21T09:22:17.400+0000 7fdc967ba640 -1 mgr.server handle_report got status from non-daemon mon.b 2024-09-21T09:22:17.888 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.0 2024-09-21T09:22:17.965 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.2 2024-09-21T09:22:17.990 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.1 2024-09-21T09:22:17.998 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:17.995+0000 7f78ff1ce640 1 -- 172.21.15.78:0/1191192410 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f78f80966f0 msgr2=0x7f78f812b720 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:17.998 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:17.995+0000 7f78ff1ce640 1 --2- 172.21.15.78:0/1191192410 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f78f80966f0 0x7f78f812b720 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7f78e4009900 tx=0x7f78e402a4d0 comp rx=0 tx=0).stop 2024-09-21T09:22:17.998 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:17.995+0000 7f78ff1ce640 1 -- 172.21.15.78:0/1191192410 shutdown_connections 2024-09-21T09:22:17.998 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:17.995+0000 7f78ff1ce640 1 --2- 172.21.15.78:0/1191192410 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f78f80966f0 0x7f78f812b720 unknown :-1 s=CLOSED pgs=19 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:17.998 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:17.995+0000 7f78ff1ce640 1 --2- 172.21.15.78:0/1191192410 >> v2:172.21.15.78:3300/0 conn(0x7f78f80973f0 0x7f78f812bc60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:17.998 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:17.995+0000 7f78ff1ce640 1 --2- 172.21.15.78:0/1191192410 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f78f8095dd0 0x7f78f80961b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:17.998 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:17.995+0000 7f78ff1ce640 1 -- 172.21.15.78:0/1191192410 >> 172.21.15.78:0/1191192410 conn(0x7f78f809c470 msgr2=0x7f78f809c880 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:17.998 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:17.995+0000 7f78ff1ce640 1 -- 172.21.15.78:0/1191192410 shutdown_connections 2024-09-21T09:22:17.998 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:17.995+0000 7f78ff1ce640 1 -- 172.21.15.78:0/1191192410 wait complete. 2024-09-21T09:22:18.000 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:17.996+0000 7f78ff1ce640 1 Processor -- start 2024-09-21T09:22:18.000 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:17.996+0000 7f78ff1ce640 1 -- start start 2024-09-21T09:22:18.000 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:17.996+0000 7f78ff1ce640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f78f8095dd0 0x7f78f80719f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:18.000 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:17.996+0000 7f78ff1ce640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f78f80966f0 0x7f78f8071f30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:18.000 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:17.996+0000 7f78ff1ce640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f78f80973f0 0x7f78f8072470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:18.000 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:17.996+0000 7f78ff1ce640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7f78f8133380 con 0x7f78f8095dd0 2024-09-21T09:22:18.000 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:17.996+0000 7f78ff1ce640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7f78f8133200 con 0x7f78f80973f0 2024-09-21T09:22:18.000 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:17.996+0000 7f78ff1ce640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7f78f8133500 con 0x7f78f80966f0 2024-09-21T09:22:18.001 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:17.997+0000 7f78fcf6a640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f78f8095dd0 0x7f78f80719f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:18.001 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:17.997+0000 7f78fd76b640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f78f80973f0 0x7f78f8072470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:18.001 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:17.997+0000 7f78fcf6a640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f78f8095dd0 0x7f78f80719f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3300/0 says I am v2:172.21.15.78:51318/0 (socket says 172.21.15.78:51318) 2024-09-21T09:22:18.001 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:17.997+0000 7f78fcf6a640 1 -- 172.21.15.78:0/998775040 learned_addr learned my addr 172.21.15.78:0/998775040 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:18.001 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:17.997+0000 7f78f7fff640 1 --2- 172.21.15.78:0/998775040 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f78f80966f0 0x7f78f8071f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:18.001 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:17.997+0000 7f78fd76b640 1 -- 172.21.15.78:0/998775040 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f78f80966f0 msgr2=0x7f78f8071f30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:18.001 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:17.997+0000 7f78fd76b640 1 --2- 172.21.15.78:0/998775040 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f78f80966f0 0x7f78f8071f30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:18.001 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:17.997+0000 7f78fd76b640 1 -- 172.21.15.78:0/998775040 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f78f8095dd0 msgr2=0x7f78f80719f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:18.001 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:17.997+0000 7f78fd76b640 1 --2- 172.21.15.78:0/998775040 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f78f8095dd0 0x7f78f80719f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:18.001 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:17.997+0000 7f78fd76b640 1 -- 172.21.15.78:0/998775040 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f78f806e540 con 0x7f78f80973f0 2024-09-21T09:22:18.001 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:17.997+0000 7f78fcf6a640 1 --2- 172.21.15.78:0/998775040 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f78f8095dd0 0x7f78f80719f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:22:18.002 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:17.997+0000 7f78fd76b640 1 --2- 172.21.15.78:0/998775040 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f78f80973f0 0x7f78f8072470 secure :-1 s=READY pgs=26 cs=0 l=1 rev1=1 crypto rx=0x7f78f0005fd0 tx=0x7f78f000c6a0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:18.002 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:17.997+0000 7f78f57fa640 1 -- 172.21.15.78:0/998775040 <== mon.1 v2:172.21.15.78:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f78f0010630 con 0x7f78f80973f0 2024-09-21T09:22:18.002 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:17.997+0000 7f78f57fa640 1 -- 172.21.15.78:0/998775040 <== mon.1 v2:172.21.15.78:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f78f00107d0 con 0x7f78f80973f0 2024-09-21T09:22:18.003 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:17.997+0000 7f78f57fa640 1 -- 172.21.15.78:0/998775040 <== mon.1 v2:172.21.15.78:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f78f0010c00 con 0x7f78f80973f0 2024-09-21T09:22:18.003 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:17.997+0000 7f78ff1ce640 1 -- 172.21.15.78:0/998775040 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f78f806ebb0 con 0x7f78f80973f0 2024-09-21T09:22:18.003 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:17.997+0000 7f78ff1ce640 1 -- 172.21.15.78:0/998775040 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7f78f80730f0 con 0x7f78f80973f0 2024-09-21T09:22:18.003 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:17.998+0000 7f78f57fa640 1 -- 172.21.15.78:0/998775040 <== mon.1 v2:172.21.15.78:3301/0 4 ==== mgrmap(e 8) ==== 63336+0+0 (secure 0 0 0) 0x7f78f0021430 con 0x7f78f80973f0 2024-09-21T09:22:18.003 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:17.999+0000 7f78ff1ce640 1 -- 172.21.15.78:0/998775040 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f78f80961b0 con 0x7f78f80973f0 2024-09-21T09:22:18.003 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:17.999+0000 7f78f57fa640 1 -- 172.21.15.78:0/998775040 <== mon.1 v2:172.21.15.78:3301/0 5 ==== osd_map(20..20 src has 1..20) ==== 3027+0+0 (secure 0 0 0) 0x7f78f0069b10 con 0x7f78f80973f0 2024-09-21T09:22:18.005 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.003+0000 7f78f57fa640 1 -- 172.21.15.78:0/998775040 <== mon.1 v2:172.21.15.78:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f78f0069e20 con 0x7f78f80973f0 2024-09-21T09:22:18.041 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.3 2024-09-21T09:22:18.072 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.070+0000 7fb65d73f640 1 -- 172.21.15.78:0/392443863 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fb658096780 msgr2=0x7fb658096bd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:18.072 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.070+0000 7fb65d73f640 1 --2- 172.21.15.78:0/392443863 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fb658096780 0x7fb658096bd0 secure :-1 s=READY pgs=53 cs=0 l=1 rev1=1 crypto rx=0x7fb650005fd0 tx=0x7fb65002c860 comp rx=0 tx=0).stop 2024-09-21T09:22:18.072 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.070+0000 7fb65d73f640 1 -- 172.21.15.78:0/392443863 shutdown_connections 2024-09-21T09:22:18.072 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.070+0000 7fb65d73f640 1 --2- 172.21.15.78:0/392443863 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fb658096780 0x7fb658096bd0 unknown :-1 s=CLOSED pgs=53 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:18.072 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.070+0000 7fb65d73f640 1 --2- 172.21.15.78:0/392443863 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fb658125b70 0x7fb658125f50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:18.072 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.070+0000 7fb65d73f640 1 -- 172.21.15.78:0/392443863 >> 172.21.15.78:0/392443863 conn(0x7fb65809e610 msgr2=0x7fb65809ea20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:18.073 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.070+0000 7fb65d73f640 1 -- 172.21.15.78:0/392443863 shutdown_connections 2024-09-21T09:22:18.073 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.070+0000 7fb65d73f640 1 -- 172.21.15.78:0/392443863 wait complete. 2024-09-21T09:22:18.073 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.071+0000 7fb65d73f640 1 Processor -- start 2024-09-21T09:22:18.073 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.071+0000 7fb65d73f640 1 -- start start 2024-09-21T09:22:18.075 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.072+0000 7fb65d73f640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fb658095c50 0x7fb65808ce80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:18.075 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.072+0000 7fb65d73f640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fb658096780 0x7fb65808d3c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:18.075 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.072+0000 7fb65d73f640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fb658125b70 0x7fb65808d900 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:18.076 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.072+0000 7fb65d73f640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7fb65806eec0 con 0x7fb658095c50 2024-09-21T09:22:18.076 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.072+0000 7fb65d73f640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7fb65806ed40 con 0x7fb658096780 2024-09-21T09:22:18.076 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.072+0000 7fb65d73f640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7fb65806f040 con 0x7fb658125b70 2024-09-21T09:22:18.076 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.072+0000 7fb6567fc640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fb658096780 0x7fb65808d3c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:18.076 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.072+0000 7fb656ffd640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fb658095c50 0x7fb65808ce80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:18.076 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.072+0000 7fb6567fc640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fb658096780 0x7fb65808d3c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3301/0 says I am v2:172.21.15.78:60826/0 (socket says 172.21.15.78:60826) 2024-09-21T09:22:18.076 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.072+0000 7fb6577fe640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fb658125b70 0x7fb65808d900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:18.076 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.072+0000 7fb6567fc640 1 -- 172.21.15.78:0/3699490038 learned_addr learned my addr 172.21.15.78:0/3699490038 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:18.076 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.072+0000 7fb6567fc640 1 -- 172.21.15.78:0/3699490038 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fb658125b70 msgr2=0x7fb65808d900 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:18.076 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.072+0000 7fb6567fc640 1 --2- 172.21.15.78:0/3699490038 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fb658125b70 0x7fb65808d900 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:18.076 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.072+0000 7fb6567fc640 1 -- 172.21.15.78:0/3699490038 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fb658095c50 msgr2=0x7fb65808ce80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:18.076 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.073+0000 7fb6567fc640 1 --2- 172.21.15.78:0/3699490038 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fb658095c50 0x7fb65808ce80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:18.076 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.073+0000 7fb6567fc640 1 -- 172.21.15.78:0/3699490038 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fb6581264f0 con 0x7fb658096780 2024-09-21T09:22:18.077 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.073+0000 7fb656ffd640 1 --2- 172.21.15.78:0/3699490038 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fb658095c50 0x7fb65808ce80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:22:18.077 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.073+0000 7fb6567fc640 1 --2- 172.21.15.78:0/3699490038 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fb658096780 0x7fb65808d3c0 secure :-1 s=READY pgs=27 cs=0 l=1 rev1=1 crypto rx=0x7fb64c00c500 tx=0x7fb64c00c9c0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:18.077 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.073+0000 7fb63f7fe640 1 -- 172.21.15.78:0/3699490038 <== mon.1 v2:172.21.15.78:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb64c0052d0 con 0x7fb658096780 2024-09-21T09:22:18.077 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.073+0000 7fb63f7fe640 1 -- 172.21.15.78:0/3699490038 <== mon.1 v2:172.21.15.78:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fb64c005470 con 0x7fb658096780 2024-09-21T09:22:18.077 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.073+0000 7fb65d73f640 1 -- 172.21.15.78:0/3699490038 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fb65808f530 con 0x7fb658096780 2024-09-21T09:22:18.077 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.073+0000 7fb63f7fe640 1 -- 172.21.15.78:0/3699490038 <== mon.1 v2:172.21.15.78:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb64c012870 con 0x7fb658096780 2024-09-21T09:22:18.077 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.073+0000 7fb65d73f640 1 -- 172.21.15.78:0/3699490038 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7fb65808eb30 con 0x7fb658096780 2024-09-21T09:22:18.078 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.074+0000 7fb63f7fe640 1 -- 172.21.15.78:0/3699490038 <== mon.1 v2:172.21.15.78:3301/0 4 ==== mgrmap(e 8) ==== 63336+0+0 (secure 0 0 0) 0x7fb64c0035f0 con 0x7fb658096780 2024-09-21T09:22:18.078 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.074+0000 7fb65d73f640 1 -- 172.21.15.78:0/3699490038 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fb65809cf90 con 0x7fb658096780 2024-09-21T09:22:18.078 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.075+0000 7fb63f7fe640 1 -- 172.21.15.78:0/3699490038 <== mon.1 v2:172.21.15.78:3301/0 5 ==== osd_map(20..20 src has 1..20) ==== 3027+0+0 (secure 0 0 0) 0x7fb64c068890 con 0x7fb658096780 2024-09-21T09:22:18.082 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.078+0000 7fb63f7fe640 1 -- 172.21.15.78:0/3699490038 <== mon.1 v2:172.21.15.78:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7fb65809cf90 con 0x7fb658096780 2024-09-21T09:22:18.112 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.109+0000 7fd776bce640 1 -- 172.21.15.78:0/3930494624 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fd770129e30 msgr2=0x7fd77012c820 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:18.112 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.109+0000 7fd776bce640 1 --2- 172.21.15.78:0/3930494624 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fd770129e30 0x7fd77012c820 secure :-1 s=READY pgs=55 cs=0 l=1 rev1=1 crypto rx=0x7fd75c005fd0 tx=0x7fd75c02a4b0 comp rx=0 tx=0).stop 2024-09-21T09:22:18.112 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.109+0000 7fd776bce640 1 -- 172.21.15.78:0/3930494624 shutdown_connections 2024-09-21T09:22:18.112 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.109+0000 7fd776bce640 1 --2- 172.21.15.78:0/3930494624 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fd770129e30 0x7fd77012c820 unknown :-1 s=CLOSED pgs=55 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:18.112 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.109+0000 7fd776bce640 1 --2- 172.21.15.78:0/3930494624 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fd770129510 0x7fd7701298f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:18.112 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.109+0000 7fd776bce640 1 -- 172.21.15.78:0/3930494624 >> 172.21.15.78:0/3930494624 conn(0x7fd77009d370 msgr2=0x7fd77009d780 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:18.112 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.110+0000 7fd776bce640 1 -- 172.21.15.78:0/3930494624 shutdown_connections 2024-09-21T09:22:18.113 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.110+0000 7fd776bce640 1 -- 172.21.15.78:0/3930494624 wait complete. 2024-09-21T09:22:18.113 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.111+0000 7fd776bce640 1 Processor -- start 2024-09-21T09:22:18.113 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.111+0000 7fd776bce640 1 -- start start 2024-09-21T09:22:18.115 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.112+0000 7fd776bce640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fd770129510 0x7fd77012ffc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:18.115 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.112+0000 7fd776bce640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fd770129e30 0x7fd770130500 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:18.116 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.112+0000 7fd776bce640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fd77012cd60 0x7fd770130a40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:18.116 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.112+0000 7fd776bce640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7fd770135750 con 0x7fd770129510 2024-09-21T09:22:18.116 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.112+0000 7fd776bce640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7fd7701355d0 con 0x7fd770129e30 2024-09-21T09:22:18.116 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.112+0000 7fd776bce640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7fd7701358d0 con 0x7fd77012cd60 2024-09-21T09:22:18.116 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.112+0000 7fd76ffff640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fd770129e30 0x7fd770130500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:18.116 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.112+0000 7fd76ffff640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fd770129e30 0x7fd770130500 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3301/0 says I am v2:172.21.15.78:60842/0 (socket says 172.21.15.78:60842) 2024-09-21T09:22:18.116 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.112+0000 7fd76ffff640 1 -- 172.21.15.78:0/1074502533 learned_addr learned my addr 172.21.15.78:0/1074502533 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:18.116 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.112+0000 7fd77516b640 1 --2- 172.21.15.78:0/1074502533 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fd77012cd60 0x7fd770130a40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:18.116 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.112+0000 7fd77496a640 1 --2- 172.21.15.78:0/1074502533 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fd770129510 0x7fd77012ffc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:18.116 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.112+0000 7fd76ffff640 1 -- 172.21.15.78:0/1074502533 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fd77012cd60 msgr2=0x7fd770130a40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:18.116 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.112+0000 7fd76ffff640 1 --2- 172.21.15.78:0/1074502533 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fd77012cd60 0x7fd770130a40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:18.116 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.112+0000 7fd76ffff640 1 -- 172.21.15.78:0/1074502533 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fd770129510 msgr2=0x7fd77012ffc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:18.116 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.112+0000 7fd76ffff640 1 --2- 172.21.15.78:0/1074502533 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fd770129510 0x7fd77012ffc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:18.117 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.112+0000 7fd76ffff640 1 -- 172.21.15.78:0/1074502533 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fd770126f70 con 0x7fd770129e30 2024-09-21T09:22:18.117 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.112+0000 7fd77516b640 1 --2- 172.21.15.78:0/1074502533 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fd77012cd60 0x7fd770130a40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:22:18.117 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.113+0000 7fd76ffff640 1 --2- 172.21.15.78:0/1074502533 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fd770129e30 0x7fd770130500 secure :-1 s=READY pgs=28 cs=0 l=1 rev1=1 crypto rx=0x7fd75c006010 tx=0x7fd75c0039e0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:18.117 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.113+0000 7fd76d7fa640 1 -- 172.21.15.78:0/1074502533 <== mon.1 v2:172.21.15.78:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd75c03f070 con 0x7fd770129e30 2024-09-21T09:22:18.117 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.113+0000 7fd76d7fa640 1 -- 172.21.15.78:0/1074502533 <== mon.1 v2:172.21.15.78:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fd75c004890 con 0x7fd770129e30 2024-09-21T09:22:18.117 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.113+0000 7fd76d7fa640 1 -- 172.21.15.78:0/1074502533 <== mon.1 v2:172.21.15.78:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd75c035030 con 0x7fd770129e30 2024-09-21T09:22:18.117 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.113+0000 7fd776bce640 1 -- 172.21.15.78:0/1074502533 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fd770131560 con 0x7fd770129e30 2024-09-21T09:22:18.117 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.113+0000 7fd776bce640 1 -- 172.21.15.78:0/1074502533 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7fd770132330 con 0x7fd770129e30 2024-09-21T09:22:18.118 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.114+0000 7fd776bce640 1 -- 172.21.15.78:0/1074502533 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fd7701298f0 con 0x7fd770129e30 2024-09-21T09:22:18.118 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.114+0000 7fd76d7fa640 1 -- 172.21.15.78:0/1074502533 <== mon.1 v2:172.21.15.78:3301/0 4 ==== mgrmap(e 8) ==== 63336+0+0 (secure 0 0 0) 0x7fd75c042070 con 0x7fd770129e30 2024-09-21T09:22:18.118 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.115+0000 7fd76d7fa640 1 -- 172.21.15.78:0/1074502533 <== mon.1 v2:172.21.15.78:3301/0 5 ==== osd_map(20..20 src has 1..20) ==== 3027+0+0 (secure 0 0 0) 0x7fd75c089c30 con 0x7fd770129e30 2024-09-21T09:22:18.120 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.118+0000 7fd76d7fa640 1 -- 172.21.15.78:0/1074502533 <== mon.1 v2:172.21.15.78:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7fd75c088990 con 0x7fd770129e30 2024-09-21T09:22:18.149 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.146+0000 7fc8a2a1d640 1 -- 172.21.15.78:0/3418714295 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fc89c098d40 msgr2=0x7fc89c099120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:18.149 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.146+0000 7fc8a2a1d640 1 --2- 172.21.15.78:0/3418714295 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fc89c098d40 0x7fc89c099120 secure :-1 s=READY pgs=20 cs=0 l=1 rev1=1 crypto rx=0x7fc88c005fd0 tx=0x7fc88c02a410 comp rx=0 tx=0).stop 2024-09-21T09:22:18.149 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.146+0000 7fc8a2a1d640 1 -- 172.21.15.78:0/3418714295 shutdown_connections 2024-09-21T09:22:18.149 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.146+0000 7fc8a2a1d640 1 --2- 172.21.15.78:0/3418714295 >> v2:172.21.15.78:3300/0 conn(0x7fc89c099660 0x7fc89c12eab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:18.149 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.146+0000 7fc8a2a1d640 1 --2- 172.21.15.78:0/3418714295 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fc89c098d40 0x7fc89c099120 unknown :-1 s=CLOSED pgs=20 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:18.149 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.146+0000 7fc8a2a1d640 1 -- 172.21.15.78:0/3418714295 >> 172.21.15.78:0/3418714295 conn(0x7fc89c0a1790 msgr2=0x7fc89c0a1ba0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:18.149 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.146+0000 7fc8a2a1d640 1 -- 172.21.15.78:0/3418714295 shutdown_connections 2024-09-21T09:22:18.149 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.146+0000 7fc8a2a1d640 1 -- 172.21.15.78:0/3418714295 wait complete. 2024-09-21T09:22:18.151 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.147+0000 7fc8a2a1d640 1 Processor -- start 2024-09-21T09:22:18.151 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.147+0000 7fc8a2a1d640 1 -- start start 2024-09-21T09:22:18.151 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.147+0000 7fc8a2a1d640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fc89c098d40 0x7fc89c1c4f90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:18.151 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.147+0000 7fc8a2a1d640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fc89c099660 0x7fc89c1c54d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:18.151 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.147+0000 7fc8a2a1d640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fc89c12eff0 0x7fc89c0712d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:18.151 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.147+0000 7fc8a2a1d640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7fc89c137870 con 0x7fc89c099660 2024-09-21T09:22:18.151 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.147+0000 7fc8a2a1d640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7fc89c1376f0 con 0x7fc89c12eff0 2024-09-21T09:22:18.151 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.147+0000 7fc8a2a1d640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7fc89c1379f0 con 0x7fc89c098d40 2024-09-21T09:22:18.152 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.148+0000 7fc89b7fe640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fc89c099660 0x7fc89c1c54d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:18.152 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.148+0000 7fc89b7fe640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fc89c099660 0x7fc89c1c54d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3300/0 says I am v2:172.21.15.78:51368/0 (socket says 172.21.15.78:51368) 2024-09-21T09:22:18.152 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.148+0000 7fc89b7fe640 1 -- 172.21.15.78:0/2216548963 learned_addr learned my addr 172.21.15.78:0/2216548963 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:18.152 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.148+0000 7fc8a0fba640 1 --2- 172.21.15.78:0/2216548963 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fc89c12eff0 0x7fc89c0712d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:18.152 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.148+0000 7fc89bfff640 1 --2- 172.21.15.78:0/2216548963 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fc89c098d40 0x7fc89c1c4f90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:18.152 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.148+0000 7fc89b7fe640 1 -- 172.21.15.78:0/2216548963 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fc89c098d40 msgr2=0x7fc89c1c4f90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:18.152 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.148+0000 7fc89b7fe640 1 --2- 172.21.15.78:0/2216548963 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fc89c098d40 0x7fc89c1c4f90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:18.152 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.148+0000 7fc89b7fe640 1 -- 172.21.15.78:0/2216548963 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fc89c12eff0 msgr2=0x7fc89c0712d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:18.152 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.148+0000 7fc89b7fe640 1 --2- 172.21.15.78:0/2216548963 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fc89c12eff0 0x7fc89c0712d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:18.152 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.148+0000 7fc89b7fe640 1 -- 172.21.15.78:0/2216548963 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fc89c073340 con 0x7fc89c099660 2024-09-21T09:22:18.152 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.148+0000 7fc89bfff640 1 --2- 172.21.15.78:0/2216548963 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fc89c098d40 0x7fc89c1c4f90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-09-21T09:22:18.152 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.148+0000 7fc89b7fe640 1 --2- 172.21.15.78:0/2216548963 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fc89c099660 0x7fc89c1c54d0 secure :-1 s=READY pgs=56 cs=0 l=1 rev1=1 crypto rx=0x7fc89000b4d0 tx=0x7fc89000b990 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:18.152 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.148+0000 7fc898ff9640 1 -- 172.21.15.78:0/2216548963 <== mon.0 v2:172.21.15.78:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc890014070 con 0x7fc89c099660 2024-09-21T09:22:18.153 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.148+0000 7fc8a2a1d640 1 -- 172.21.15.78:0/2216548963 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fc89c129460 con 0x7fc89c099660 2024-09-21T09:22:18.153 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.148+0000 7fc898ff9640 1 -- 172.21.15.78:0/2216548963 <== mon.0 v2:172.21.15.78:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fc890010960 con 0x7fc89c099660 2024-09-21T09:22:18.153 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.148+0000 7fc8a2a1d640 1 -- 172.21.15.78:0/2216548963 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fc89c1c5a10 con 0x7fc89c099660 2024-09-21T09:22:18.153 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.148+0000 7fc898ff9640 1 -- 172.21.15.78:0/2216548963 <== mon.0 v2:172.21.15.78:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc890010db0 con 0x7fc89c099660 2024-09-21T09:22:18.153 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.150+0000 7fc898ff9640 1 -- 172.21.15.78:0/2216548963 <== mon.0 v2:172.21.15.78:3300/0 4 ==== mgrmap(e 8) ==== 63336+0+0 (secure 0 0 0) 0x7fc890017070 con 0x7fc89c099660 2024-09-21T09:22:18.153 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.149+0000 7fc8a2a1d640 1 -- 172.21.15.78:0/2216548963 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fc89c099120 con 0x7fc89c099660 2024-09-21T09:22:18.153 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.150+0000 7fc898ff9640 1 -- 172.21.15.78:0/2216548963 <== mon.0 v2:172.21.15.78:3300/0 5 ==== osd_map(20..20 src has 1..20) ==== 3027+0+0 (secure 0 0 0) 0x7fc890068b10 con 0x7fc89c099660 2024-09-21T09:22:18.156 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.154+0000 7fc898ff9640 1 -- 172.21.15.78:0/2216548963 <== mon.0 v2:172.21.15.78:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7fc890067940 con 0x7fc89c099660 2024-09-21T09:22:18.168 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.166+0000 7f78ff1ce640 1 -- 172.21.15.78:0/998775040 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 0} v 0) -- 0x7f78f81c27f0 con 0x7f78f80973f0 2024-09-21T09:22:18.168 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.166+0000 7f78f57fa640 1 -- 172.21.15.78:0/998775040 <== mon.1 v2:172.21.15.78:3301/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 0}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7f78f00450c0 con 0x7f78f80973f0 2024-09-21T09:22:18.169 INFO:teuthology.orchestra.run.smithi078.stdout:38654705682 2024-09-21T09:22:18.171 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.168+0000 7f78ff1ce640 1 -- 172.21.15.78:0/998775040 >> v1:172.21.15.78:6816/2186157269 conn(0x7f78dc051300 legacy=0x7f78dc0537c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:18.171 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.168+0000 7f78ff1ce640 1 -- 172.21.15.78:0/998775040 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f78f80973f0 msgr2=0x7f78f8072470 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:18.171 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.168+0000 7f78ff1ce640 1 --2- 172.21.15.78:0/998775040 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f78f80973f0 0x7f78f8072470 secure :-1 s=READY pgs=26 cs=0 l=1 rev1=1 crypto rx=0x7f78f0005fd0 tx=0x7f78f000c6a0 comp rx=0 tx=0).stop 2024-09-21T09:22:18.171 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.168+0000 7f78ff1ce640 1 -- 172.21.15.78:0/998775040 shutdown_connections 2024-09-21T09:22:18.171 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.168+0000 7f78ff1ce640 1 --2- 172.21.15.78:0/998775040 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f78f80973f0 0x7f78f8072470 unknown :-1 s=CLOSED pgs=26 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:18.171 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.168+0000 7f78ff1ce640 1 --2- 172.21.15.78:0/998775040 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f78f80966f0 0x7f78f8071f30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:18.171 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.168+0000 7f78ff1ce640 1 --2- 172.21.15.78:0/998775040 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f78f8095dd0 0x7f78f80719f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:18.171 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.168+0000 7f78ff1ce640 1 -- 172.21.15.78:0/998775040 >> 172.21.15.78:0/998775040 conn(0x7f78f809c470 msgr2=0x7f78f8125220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:18.171 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.169+0000 7f78ff1ce640 1 -- 172.21.15.78:0/998775040 shutdown_connections 2024-09-21T09:22:18.171 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.169+0000 7f78ff1ce640 1 -- 172.21.15.78:0/998775040 wait complete. 2024-09-21T09:22:18.182 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705683 got 38654705682 for osd.0 2024-09-21T09:22:18.304 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.301+0000 7fb65d73f640 1 -- 172.21.15.78:0/3699490038 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 2} v 0) -- 0x7fb658096bd0 con 0x7fb658096780 2024-09-21T09:22:18.304 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.301+0000 7fb63f7fe640 1 -- 172.21.15.78:0/3699490038 <== mon.1 v2:172.21.15.78:3301/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 2}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7fb64c0676c0 con 0x7fb658096780 2024-09-21T09:22:18.304 INFO:teuthology.orchestra.run.smithi078.stdout:38654705684 2024-09-21T09:22:18.306 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.304+0000 7fb65d73f640 1 -- 172.21.15.78:0/3699490038 >> v1:172.21.15.78:6816/2186157269 conn(0x7fb644051300 legacy=0x7fb6440537c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:18.306 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.304+0000 7fb65d73f640 1 -- 172.21.15.78:0/3699490038 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fb658096780 msgr2=0x7fb65808d3c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:18.306 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.304+0000 7fb65d73f640 1 --2- 172.21.15.78:0/3699490038 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fb658096780 0x7fb65808d3c0 secure :-1 s=READY pgs=27 cs=0 l=1 rev1=1 crypto rx=0x7fb64c00c500 tx=0x7fb64c00c9c0 comp rx=0 tx=0).stop 2024-09-21T09:22:18.307 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.304+0000 7fb65d73f640 1 -- 172.21.15.78:0/3699490038 shutdown_connections 2024-09-21T09:22:18.307 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.304+0000 7fb65d73f640 1 --2- 172.21.15.78:0/3699490038 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fb658125b70 0x7fb65808d900 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:18.307 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.304+0000 7fb65d73f640 1 --2- 172.21.15.78:0/3699490038 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fb658096780 0x7fb65808d3c0 unknown :-1 s=CLOSED pgs=27 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:18.307 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.304+0000 7fb65d73f640 1 --2- 172.21.15.78:0/3699490038 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fb658095c50 0x7fb65808ce80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:18.307 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.304+0000 7fb65d73f640 1 -- 172.21.15.78:0/3699490038 >> 172.21.15.78:0/3699490038 conn(0x7fb65809e610 msgr2=0x7fb65809ac00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:18.307 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.304+0000 7fb65d73f640 1 -- 172.21.15.78:0/3699490038 shutdown_connections 2024-09-21T09:22:18.307 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.304+0000 7fb65d73f640 1 -- 172.21.15.78:0/3699490038 wait complete. 2024-09-21T09:22:18.312 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.310+0000 7fc8a2a1d640 1 -- 172.21.15.78:0/2216548963 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 3} v 0) -- 0x7fc89c0698c0 con 0x7fc89c099660 2024-09-21T09:22:18.313 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.310+0000 7fc898ff9640 1 -- 172.21.15.78:0/2216548963 <== mon.0 v2:172.21.15.78:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 3}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7fc890041910 con 0x7fc89c099660 2024-09-21T09:22:18.313 INFO:teuthology.orchestra.run.smithi078.stdout:38654705682 2024-09-21T09:22:18.315 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.312+0000 7fc8a2a1d640 1 -- 172.21.15.78:0/2216548963 >> v1:172.21.15.78:6816/2186157269 conn(0x7fc888051330 legacy=0x7fc8880537f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:18.315 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.312+0000 7fc8a2a1d640 1 -- 172.21.15.78:0/2216548963 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fc89c099660 msgr2=0x7fc89c1c54d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:18.315 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.312+0000 7fc8a2a1d640 1 --2- 172.21.15.78:0/2216548963 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fc89c099660 0x7fc89c1c54d0 secure :-1 s=READY pgs=56 cs=0 l=1 rev1=1 crypto rx=0x7fc89000b4d0 tx=0x7fc89000b990 comp rx=0 tx=0).stop 2024-09-21T09:22:18.316 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.312+0000 7fc8a2a1d640 1 -- 172.21.15.78:0/2216548963 shutdown_connections 2024-09-21T09:22:18.316 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.312+0000 7fc8a2a1d640 1 --2- 172.21.15.78:0/2216548963 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fc89c12eff0 0x7fc89c0712d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:18.316 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.312+0000 7fc8a2a1d640 1 --2- 172.21.15.78:0/2216548963 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fc89c099660 0x7fc89c1c54d0 unknown :-1 s=CLOSED pgs=56 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:18.316 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.312+0000 7fc8a2a1d640 1 --2- 172.21.15.78:0/2216548963 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fc89c098d40 0x7fc89c1c4f90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:18.316 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.312+0000 7fc8a2a1d640 1 -- 172.21.15.78:0/2216548963 >> 172.21.15.78:0/2216548963 conn(0x7fc89c0a1790 msgr2=0x7fc89c09cd10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:18.316 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.313+0000 7fc8a2a1d640 1 -- 172.21.15.78:0/2216548963 shutdown_connections 2024-09-21T09:22:18.316 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.313+0000 7fc8a2a1d640 1 -- 172.21.15.78:0/2216548963 wait complete. 2024-09-21T09:22:18.317 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705685 got 38654705684 for osd.2 2024-09-21T09:22:18.325 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705683 got 38654705682 for osd.3 2024-09-21T09:22:18.334 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.332+0000 7fd776bce640 1 -- 172.21.15.78:0/1074502533 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 1} v 0) -- 0x7fd770132700 con 0x7fd770129e30 2024-09-21T09:22:18.334 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.332+0000 7fd76d7fa640 1 -- 172.21.15.78:0/1074502533 <== mon.1 v2:172.21.15.78:3301/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 1}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7fd75c062960 con 0x7fd770129e30 2024-09-21T09:22:18.335 INFO:teuthology.orchestra.run.smithi078.stdout:38654705682 2024-09-21T09:22:18.337 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.334+0000 7fd776bce640 1 -- 172.21.15.78:0/1074502533 >> v1:172.21.15.78:6816/2186157269 conn(0x7fd754051330 legacy=0x7fd7540537f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:18.337 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.334+0000 7fd776bce640 1 -- 172.21.15.78:0/1074502533 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fd770129e30 msgr2=0x7fd770130500 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:18.337 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.334+0000 7fd776bce640 1 --2- 172.21.15.78:0/1074502533 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fd770129e30 0x7fd770130500 secure :-1 s=READY pgs=28 cs=0 l=1 rev1=1 crypto rx=0x7fd75c006010 tx=0x7fd75c0039e0 comp rx=0 tx=0).stop 2024-09-21T09:22:18.337 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.334+0000 7fd776bce640 1 -- 172.21.15.78:0/1074502533 shutdown_connections 2024-09-21T09:22:18.337 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.334+0000 7fd776bce640 1 --2- 172.21.15.78:0/1074502533 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fd77012cd60 0x7fd770130a40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:18.337 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.334+0000 7fd776bce640 1 --2- 172.21.15.78:0/1074502533 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fd770129e30 0x7fd770130500 unknown :-1 s=CLOSED pgs=28 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:18.337 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.334+0000 7fd776bce640 1 --2- 172.21.15.78:0/1074502533 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fd770129510 0x7fd77012ffc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:18.338 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.334+0000 7fd776bce640 1 -- 172.21.15.78:0/1074502533 >> 172.21.15.78:0/1074502533 conn(0x7fd77009d370 msgr2=0x7fd770128110 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:18.338 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.335+0000 7fd776bce640 1 -- 172.21.15.78:0/1074502533 shutdown_connections 2024-09-21T09:22:18.338 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:18.335+0000 7fd776bce640 1 -- 172.21.15.78:0/1074502533 wait complete. 2024-09-21T09:22:18.347 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705683 got 38654705682 for osd.1 2024-09-21T09:22:19.182 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.0 2024-09-21T09:22:19.287 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.284+0000 7fb5c09a5640 1 -- 172.21.15.78:0/3140604483 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fb5bc12eb30 msgr2=0x7fb5bc130f20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:19.287 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.284+0000 7fb5c09a5640 1 --2- 172.21.15.78:0/3140604483 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fb5bc12eb30 0x7fb5bc130f20 secure :-1 s=READY pgs=29 cs=0 l=1 rev1=1 crypto rx=0x7fb5b0005fd0 tx=0x7fb5b002a5d0 comp rx=0 tx=0).stop 2024-09-21T09:22:19.287 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.285+0000 7fb5c09a5640 1 -- 172.21.15.78:0/3140604483 shutdown_connections 2024-09-21T09:22:19.287 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.285+0000 7fb5c09a5640 1 --2- 172.21.15.78:0/3140604483 >> v2:172.21.15.78:3300/0 conn(0x7fb5bc131460 0x7fb5bc133850 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:19.287 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.285+0000 7fb5c09a5640 1 --2- 172.21.15.78:0/3140604483 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fb5bc12eb30 0x7fb5bc130f20 unknown :-1 s=CLOSED pgs=29 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:19.288 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.285+0000 7fb5c09a5640 1 -- 172.21.15.78:0/3140604483 >> 172.21.15.78:0/3140604483 conn(0x7fb5bc09afd0 msgr2=0x7fb5bc09b3e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:19.288 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.285+0000 7fb5c09a5640 1 -- 172.21.15.78:0/3140604483 shutdown_connections 2024-09-21T09:22:19.288 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.285+0000 7fb5c09a5640 1 -- 172.21.15.78:0/3140604483 wait complete. 2024-09-21T09:22:19.289 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.286+0000 7fb5c09a5640 1 Processor -- start 2024-09-21T09:22:19.290 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.286+0000 7fb5c09a5640 1 -- start start 2024-09-21T09:22:19.290 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.286+0000 7fb5c09a5640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fb5bc12bad0 0x7fb5bc08bf80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:19.290 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.286+0000 7fb5c09a5640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fb5bc12eb30 0x7fb5bc08c4c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:19.290 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.286+0000 7fb5c09a5640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fb5bc131460 0x7fb5bc08ca00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:19.290 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.286+0000 7fb5c09a5640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7fb5bc134680 con 0x7fb5bc131460 2024-09-21T09:22:19.290 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.286+0000 7fb5c09a5640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7fb5bc134500 con 0x7fb5bc12eb30 2024-09-21T09:22:19.290 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.286+0000 7fb5c09a5640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7fb5bc134800 con 0x7fb5bc12bad0 2024-09-21T09:22:19.290 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.286+0000 7fb5b9d9b640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fb5bc12eb30 0x7fb5bc08c4c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:19.290 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.286+0000 7fb5ba59c640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fb5bc12bad0 0x7fb5bc08bf80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:19.290 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.286+0000 7fb5b9d9b640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fb5bc12eb30 0x7fb5bc08c4c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3301/0 says I am v2:172.21.15.78:60894/0 (socket says 172.21.15.78:60894) 2024-09-21T09:22:19.290 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.286+0000 7fb5bad9d640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fb5bc131460 0x7fb5bc08ca00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:19.290 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.286+0000 7fb5b9d9b640 1 -- 172.21.15.78:0/4193439028 learned_addr learned my addr 172.21.15.78:0/4193439028 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:19.291 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.286+0000 7fb5b9d9b640 1 -- 172.21.15.78:0/4193439028 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fb5bc12bad0 msgr2=0x7fb5bc08bf80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:19.291 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.286+0000 7fb5b9d9b640 1 --2- 172.21.15.78:0/4193439028 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fb5bc12bad0 0x7fb5bc08bf80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:19.291 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.286+0000 7fb5b9d9b640 1 -- 172.21.15.78:0/4193439028 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fb5bc131460 msgr2=0x7fb5bc08ca00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:19.291 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.286+0000 7fb5b9d9b640 1 --2- 172.21.15.78:0/4193439028 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fb5bc131460 0x7fb5bc08ca00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:19.291 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.286+0000 7fb5b9d9b640 1 -- 172.21.15.78:0/4193439028 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fb5bc08e190 con 0x7fb5bc12eb30 2024-09-21T09:22:19.291 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.286+0000 7fb5ba59c640 1 --2- 172.21.15.78:0/4193439028 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fb5bc12bad0 0x7fb5bc08bf80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:22:19.291 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.286+0000 7fb5bad9d640 1 --2- 172.21.15.78:0/4193439028 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fb5bc131460 0x7fb5bc08ca00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:22:19.291 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.287+0000 7fb5b9d9b640 1 --2- 172.21.15.78:0/4193439028 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fb5bc12eb30 0x7fb5bc08c4c0 secure :-1 s=READY pgs=30 cs=0 l=1 rev1=1 crypto rx=0x7fb5b0000c00 tx=0x7fb5b0005770 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:19.291 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.287+0000 7fb5aaffd640 1 -- 172.21.15.78:0/4193439028 <== mon.1 v2:172.21.15.78:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb5b0035030 con 0x7fb5bc12eb30 2024-09-21T09:22:19.291 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.287+0000 7fb5aaffd640 1 -- 172.21.15.78:0/4193439028 <== mon.1 v2:172.21.15.78:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fb5b003f070 con 0x7fb5bc12eb30 2024-09-21T09:22:19.291 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.287+0000 7fb5c09a5640 1 -- 172.21.15.78:0/4193439028 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fb5bc08dbf0 con 0x7fb5bc12eb30 2024-09-21T09:22:19.291 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.287+0000 7fb5aaffd640 1 -- 172.21.15.78:0/4193439028 <== mon.1 v2:172.21.15.78:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb5b00047a0 con 0x7fb5bc12eb30 2024-09-21T09:22:19.291 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.287+0000 7fb5c09a5640 1 -- 172.21.15.78:0/4193439028 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7fb5bc133540 con 0x7fb5bc12eb30 2024-09-21T09:22:19.292 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.288+0000 7fb5c09a5640 1 -- 172.21.15.78:0/4193439028 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fb5bc12beb0 con 0x7fb5bc12eb30 2024-09-21T09:22:19.292 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.288+0000 7fb5aaffd640 1 -- 172.21.15.78:0/4193439028 <== mon.1 v2:172.21.15.78:3301/0 4 ==== mgrmap(e 9) ==== 63442+0+0 (secure 0 0 0) 0x7fb5b003b560 con 0x7fb5bc12eb30 2024-09-21T09:22:19.292 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.288+0000 7fb5aaffd640 1 -- 172.21.15.78:0/4193439028 <== mon.1 v2:172.21.15.78:3301/0 5 ==== osd_map(20..20 src has 1..20) ==== 3027+0+0 (secure 0 0 0) 0x7fb5b008b920 con 0x7fb5bc12eb30 2024-09-21T09:22:19.294 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.292+0000 7fb5aaffd640 1 -- 172.21.15.78:0/4193439028 <== mon.1 v2:172.21.15.78:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7fb5b008a200 con 0x7fb5bc12eb30 2024-09-21T09:22:19.318 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.2 2024-09-21T09:22:19.326 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.3 2024-09-21T09:22:19.348 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.1 2024-09-21T09:22:19.424 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.422+0000 7fcf9538a640 1 -- 172.21.15.78:0/2014522990 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fcf90072950 msgr2=0x7fcf9006fde0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:19.424 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.422+0000 7fcf9538a640 1 --2- 172.21.15.78:0/2014522990 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fcf90072950 0x7fcf9006fde0 secure :-1 s=READY pgs=21 cs=0 l=1 rev1=1 crypto rx=0x7fcf84005fd0 tx=0x7fcf8402a4b0 comp rx=0 tx=0).stop 2024-09-21T09:22:19.424 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.422+0000 7fcf9538a640 1 -- 172.21.15.78:0/2014522990 shutdown_connections 2024-09-21T09:22:19.425 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.422+0000 7fcf9538a640 1 --2- 172.21.15.78:0/2014522990 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fcf90072950 0x7fcf9006fde0 unknown :-1 s=CLOSED pgs=21 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:19.425 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.422+0000 7fcf9538a640 1 --2- 172.21.15.78:0/2014522990 >> v2:172.21.15.78:3300/0 conn(0x7fcf9006f4c0 0x7fcf9006f8a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:19.425 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.422+0000 7fcf9538a640 1 -- 172.21.15.78:0/2014522990 >> 172.21.15.78:0/2014522990 conn(0x7fcf9009dec0 msgr2=0x7fcf9009e2d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:19.425 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.422+0000 7fcf9538a640 1 -- 172.21.15.78:0/2014522990 shutdown_connections 2024-09-21T09:22:19.425 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.422+0000 7fcf9538a640 1 -- 172.21.15.78:0/2014522990 wait complete. 2024-09-21T09:22:19.426 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.423+0000 7fcf9538a640 1 Processor -- start 2024-09-21T09:22:19.426 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.423+0000 7fcf9538a640 1 -- start start 2024-09-21T09:22:19.426 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.423+0000 7fcf9538a640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fcf9006f4c0 0x7fcf9012ce70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:19.426 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.423+0000 7fcf9538a640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fcf90072950 0x7fcf9012d3b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:19.426 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.423+0000 7fcf9538a640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fcf90097030 0x7fcf9008ffb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:19.426 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.424+0000 7fcf8effd640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fcf9006f4c0 0x7fcf9012ce70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:19.426 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.424+0000 7fcf9538a640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7fcf90135a00 con 0x7fcf90072950 2024-09-21T09:22:19.426 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.424+0000 7fcf8effd640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fcf9006f4c0 0x7fcf9012ce70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3302/0 says I am v2:172.21.15.78:53966/0 (socket says 172.21.15.78:53966) 2024-09-21T09:22:19.426 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.424+0000 7fcf9538a640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7fcf90135880 con 0x7fcf90097030 2024-09-21T09:22:19.427 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.424+0000 7fcf8effd640 1 -- 172.21.15.78:0/1324123725 learned_addr learned my addr 172.21.15.78:0/1324123725 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:19.427 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.424+0000 7fcf9538a640 1 -- 172.21.15.78:0/1324123725 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7fcf90135b80 con 0x7fcf9006f4c0 2024-09-21T09:22:19.427 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.424+0000 7fcf8e7fc640 1 --2- 172.21.15.78:0/1324123725 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fcf90072950 0x7fcf9012d3b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:19.427 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.424+0000 7fcf8f7fe640 1 --2- 172.21.15.78:0/1324123725 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fcf90097030 0x7fcf9008ffb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:19.428 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.424+0000 7fcf8e7fc640 1 -- 172.21.15.78:0/1324123725 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fcf9006f4c0 msgr2=0x7fcf9012ce70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:19.428 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.424+0000 7fcf8e7fc640 1 --2- 172.21.15.78:0/1324123725 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fcf9006f4c0 0x7fcf9012ce70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:19.428 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.424+0000 7fcf8e7fc640 1 -- 172.21.15.78:0/1324123725 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fcf90097030 msgr2=0x7fcf9008ffb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:19.428 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.424+0000 7fcf8e7fc640 1 --2- 172.21.15.78:0/1324123725 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fcf90097030 0x7fcf9008ffb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:19.428 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.424+0000 7fcf8e7fc640 1 -- 172.21.15.78:0/1324123725 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fcf900934b0 con 0x7fcf90072950 2024-09-21T09:22:19.428 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.424+0000 7fcf8e7fc640 1 --2- 172.21.15.78:0/1324123725 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fcf90072950 0x7fcf9012d3b0 secure :-1 s=READY pgs=61 cs=0 l=1 rev1=1 crypto rx=0x7fcf90073950 tx=0x7fcf84003820 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:19.428 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.424+0000 7fcf777fe640 1 -- 172.21.15.78:0/1324123725 <== mon.0 v2:172.21.15.78:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fcf8403f070 con 0x7fcf90072950 2024-09-21T09:22:19.428 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.425+0000 7fcf777fe640 1 -- 172.21.15.78:0/1324123725 <== mon.0 v2:172.21.15.78:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fcf840047a0 con 0x7fcf90072950 2024-09-21T09:22:19.428 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.425+0000 7fcf9538a640 1 -- 172.21.15.78:0/1324123725 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fcf90092900 con 0x7fcf90072950 2024-09-21T09:22:19.428 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.425+0000 7fcf777fe640 1 -- 172.21.15.78:0/1324123725 <== mon.0 v2:172.21.15.78:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fcf84035030 con 0x7fcf90072950 2024-09-21T09:22:19.428 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.425+0000 7fcf9538a640 1 -- 172.21.15.78:0/1324123725 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fcf900904f0 con 0x7fcf90072950 2024-09-21T09:22:19.429 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.426+0000 7fcf9538a640 1 -- 172.21.15.78:0/1324123725 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fcf90099810 con 0x7fcf90072950 2024-09-21T09:22:19.429 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.426+0000 7fcf777fe640 1 -- 172.21.15.78:0/1324123725 <== mon.0 v2:172.21.15.78:3300/0 4 ==== mgrmap(e 9) ==== 63442+0+0 (secure 0 0 0) 0x7fcf84042070 con 0x7fcf90072950 2024-09-21T09:22:19.429 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.427+0000 7fcf777fe640 1 -- 172.21.15.78:0/1324123725 <== mon.0 v2:172.21.15.78:3300/0 5 ==== osd_map(20..20 src has 1..20) ==== 3027+0+0 (secure 0 0 0) 0x7fcf84089db0 con 0x7fcf90072950 2024-09-21T09:22:19.436 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.434+0000 7fcf777fe640 1 -- 172.21.15.78:0/1324123725 <== mon.0 v2:172.21.15.78:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7fcf90099810 con 0x7fcf90072950 2024-09-21T09:22:19.457 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.455+0000 7fb5c09a5640 1 -- 172.21.15.78:0/4193439028 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 0} v 0) -- 0x7fb5bc0699c0 con 0x7fb5bc12eb30 2024-09-21T09:22:19.458 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.456+0000 7fb5aaffd640 1 -- 172.21.15.78:0/4193439028 <== mon.1 v2:172.21.15.78:3301/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 0}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7fb5b008a580 con 0x7fb5bc12eb30 2024-09-21T09:22:19.458 INFO:teuthology.orchestra.run.smithi078.stdout:38654705684 2024-09-21T09:22:19.461 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.458+0000 7fb5c09a5640 1 -- 172.21.15.78:0/4193439028 >> v1:172.21.15.78:6816/2186157269 conn(0x7fb59c051420 legacy=0x7fb59c0538e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:19.461 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.458+0000 7fb5c09a5640 1 -- 172.21.15.78:0/4193439028 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fb5bc12eb30 msgr2=0x7fb5bc08c4c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:19.461 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.458+0000 7fb5c09a5640 1 --2- 172.21.15.78:0/4193439028 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fb5bc12eb30 0x7fb5bc08c4c0 secure :-1 s=READY pgs=30 cs=0 l=1 rev1=1 crypto rx=0x7fb5b0000c00 tx=0x7fb5b0005770 comp rx=0 tx=0).stop 2024-09-21T09:22:19.461 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.458+0000 7fb5c09a5640 1 -- 172.21.15.78:0/4193439028 shutdown_connections 2024-09-21T09:22:19.461 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.458+0000 7fb5c09a5640 1 --2- 172.21.15.78:0/4193439028 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fb5bc131460 0x7fb5bc08ca00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:19.461 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.458+0000 7fb5c09a5640 1 --2- 172.21.15.78:0/4193439028 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fb5bc12eb30 0x7fb5bc08c4c0 unknown :-1 s=CLOSED pgs=30 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:19.461 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.458+0000 7fb5c09a5640 1 --2- 172.21.15.78:0/4193439028 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fb5bc12bad0 0x7fb5bc08bf80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:19.461 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.458+0000 7fb5c09a5640 1 -- 172.21.15.78:0/4193439028 >> 172.21.15.78:0/4193439028 conn(0x7fb5bc09afd0 msgr2=0x7fb5bc099350 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:19.461 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.458+0000 7fb5c09a5640 1 -- 172.21.15.78:0/4193439028 shutdown_connections 2024-09-21T09:22:19.461 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.458+0000 7fb5c09a5640 1 -- 172.21.15.78:0/4193439028 wait complete. 2024-09-21T09:22:19.468 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.466+0000 7f21df59e640 1 -- 172.21.15.78:0/2635385224 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f21e006fd10 msgr2=0x7f21e012ae80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:19.468 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.466+0000 7f21df59e640 1 --2- 172.21.15.78:0/2635385224 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f21e006fd10 0x7f21e012ae80 secure :-1 s=READY pgs=22 cs=0 l=1 rev1=1 crypto rx=0x7f21d4005fd0 tx=0x7f21d402b870 comp rx=0 tx=0).stop 2024-09-21T09:22:19.468 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.466+0000 7f532bc9a640 1 -- 172.21.15.78:0/1072986993 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f5324123da0 msgr2=0x7f5324131e30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:19.468 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.466+0000 7f532bc9a640 1 --2- 172.21.15.78:0/1072986993 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f5324123da0 0x7f5324131e30 secure :-1 s=READY pgs=31 cs=0 l=1 rev1=1 crypto rx=0x7f5314009860 tx=0x7f531402a460 comp rx=0 tx=0).stop 2024-09-21T09:22:19.469 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.466+0000 7f532bc9a640 1 -- 172.21.15.78:0/1072986993 shutdown_connections 2024-09-21T09:22:19.469 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.466+0000 7f532bc9a640 1 --2- 172.21.15.78:0/1072986993 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f5324123da0 0x7f5324131e30 unknown :-1 s=CLOSED pgs=31 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:19.469 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.466+0000 7f532bc9a640 1 --2- 172.21.15.78:0/1072986993 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f5324132370 0x7f5324134760 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:19.469 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.466+0000 7f532bc9a640 1 -- 172.21.15.78:0/1072986993 >> 172.21.15.78:0/1072986993 conn(0x7f532409e610 msgr2=0x7f532409ea20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:19.469 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.466+0000 7f21df59e640 1 -- 172.21.15.78:0/2635385224 shutdown_connections 2024-09-21T09:22:19.469 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.466+0000 7f21df59e640 1 --2- 172.21.15.78:0/2635385224 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f21e006fd10 0x7f21e012ae80 unknown :-1 s=CLOSED pgs=22 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:19.469 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.466+0000 7f21df59e640 1 --2- 172.21.15.78:0/2635385224 >> v2:172.21.15.78:3300/0 conn(0x7f21e006f3f0 0x7f21e006f7d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:19.470 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.466+0000 7f21df59e640 1 -- 172.21.15.78:0/2635385224 >> 172.21.15.78:0/2635385224 conn(0x7f21e0098530 msgr2=0x7f21e0098940 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:19.470 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.466+0000 7f21df59e640 1 -- 172.21.15.78:0/2635385224 shutdown_connections 2024-09-21T09:22:19.470 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.466+0000 7f21df59e640 1 -- 172.21.15.78:0/2635385224 wait complete. 2024-09-21T09:22:19.470 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.466+0000 7f532bc9a640 1 -- 172.21.15.78:0/1072986993 shutdown_connections 2024-09-21T09:22:19.470 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.466+0000 7f532bc9a640 1 -- 172.21.15.78:0/1072986993 wait complete. 2024-09-21T09:22:19.470 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.467+0000 7f532bc9a640 1 Processor -- start 2024-09-21T09:22:19.470 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.467+0000 7f532bc9a640 1 -- start start 2024-09-21T09:22:19.471 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.467+0000 7f21df59e640 1 Processor -- start 2024-09-21T09:22:19.471 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.467+0000 7f21df59e640 1 -- start start 2024-09-21T09:22:19.472 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.468+0000 7f532bc9a640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f53241232b0 0x7f53241c0760 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:19.472 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.468+0000 7f532bc9a640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f5324123da0 0x7f53241c0ca0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:19.472 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.468+0000 7f532bc9a640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f5324132370 0x7f532412de10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:19.472 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.468+0000 7f532bc9a640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7f5324096690 con 0x7f5324132370 2024-09-21T09:22:19.472 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.468+0000 7f532bc9a640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7f5324096510 con 0x7f5324123da0 2024-09-21T09:22:19.473 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.468+0000 7f532bc9a640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7f5324096810 con 0x7f53241232b0 2024-09-21T09:22:19.473 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.468+0000 7f5329a36640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f53241232b0 0x7f53241c0760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:19.473 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.468+0000 7f5329235640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f5324123da0 0x7f53241c0ca0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:19.473 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.468+0000 7f5329a36640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f53241232b0 0x7f53241c0760 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3302/0 says I am v2:172.21.15.78:54008/0 (socket says 172.21.15.78:54008) 2024-09-21T09:22:19.473 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.468+0000 7f5329a36640 1 -- 172.21.15.78:0/103706278 learned_addr learned my addr 172.21.15.78:0/103706278 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:19.473 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.468+0000 7f5329235640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f5324123da0 0x7f53241c0ca0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3301/0 says I am v2:172.21.15.78:60926/0 (socket says 172.21.15.78:60926) 2024-09-21T09:22:19.473 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.468+0000 7f532a237640 1 --2- 172.21.15.78:0/103706278 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f5324132370 0x7f532412de10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:19.473 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.468+0000 7f532a237640 1 -- 172.21.15.78:0/103706278 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f53241232b0 msgr2=0x7f53241c0760 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:19.473 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.468+0000 7f532a237640 1 --2- 172.21.15.78:0/103706278 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f53241232b0 0x7f53241c0760 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:19.473 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.468+0000 7f532a237640 1 -- 172.21.15.78:0/103706278 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f5324123da0 msgr2=0x7f53241c0ca0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:19.473 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.468+0000 7f532a237640 1 --2- 172.21.15.78:0/103706278 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f5324123da0 0x7f53241c0ca0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:19.473 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.468+0000 7f21df59e640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f21e006f3f0 0x7f21e01c2220 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:19.474 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.468+0000 7f21df59e640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f21e006fd10 0x7f21e01c2760 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:19.474 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.468+0000 7f21df59e640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f21e012b3c0 0x7f21e01c2ca0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:19.474 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.468+0000 7f532a237640 1 -- 172.21.15.78:0/103706278 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f5324131230 con 0x7f5324132370 2024-09-21T09:22:19.474 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.468+0000 7f5329a36640 1 --2- 172.21.15.78:0/103706278 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f53241232b0 0x7f53241c0760 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:22:19.474 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.469+0000 7f21df59e640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7f21e0133550 con 0x7f21e012b3c0 2024-09-21T09:22:19.474 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.469+0000 7f21df59e640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7f21e01333d0 con 0x7f21e006f3f0 2024-09-21T09:22:19.475 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.469+0000 7f21df59e640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7f21e01336d0 con 0x7f21e006fd10 2024-09-21T09:22:19.475 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.469+0000 7f21de59c640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f21e006f3f0 0x7f21e01c2220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:19.475 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.469+0000 7f21ded9d640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f21e012b3c0 0x7f21e01c2ca0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:19.475 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.469+0000 7f21de59c640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f21e006f3f0 0x7f21e01c2220 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3301/0 says I am v2:172.21.15.78:60938/0 (socket says 172.21.15.78:60938) 2024-09-21T09:22:19.475 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.469+0000 7f21ded9d640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f21e012b3c0 0x7f21e01c2ca0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3300/0 says I am v2:172.21.15.78:51446/0 (socket says 172.21.15.78:51446) 2024-09-21T09:22:19.475 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.469+0000 7f21ded9d640 1 -- 172.21.15.78:0/1241159870 learned_addr learned my addr 172.21.15.78:0/1241159870 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:19.475 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.469+0000 7f532a237640 1 --2- 172.21.15.78:0/103706278 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f5324132370 0x7f532412de10 secure :-1 s=READY pgs=64 cs=0 l=1 rev1=1 crypto rx=0x7f531c012f90 tx=0x7f531c010490 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:19.476 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.469+0000 7f21ddd9b640 1 --2- 172.21.15.78:0/1241159870 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f21e006fd10 0x7f21e01c2760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:19.476 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.469+0000 7f21de59c640 1 -- 172.21.15.78:0/1241159870 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f21e006fd10 msgr2=0x7f21e01c2760 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:19.476 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.469+0000 7f21de59c640 1 --2- 172.21.15.78:0/1241159870 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f21e006fd10 0x7f21e01c2760 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:19.476 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.469+0000 7f21de59c640 1 -- 172.21.15.78:0/1241159870 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f21e012b3c0 msgr2=0x7f21e01c2ca0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:19.476 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.469+0000 7f21de59c640 1 --2- 172.21.15.78:0/1241159870 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f21e012b3c0 0x7f21e01c2ca0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:19.476 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.469+0000 7f21de59c640 1 -- 172.21.15.78:0/1241159870 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f21e012fa70 con 0x7f21e006f3f0 2024-09-21T09:22:19.476 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.469+0000 7f531a7fc640 1 -- 172.21.15.78:0/103706278 <== mon.0 v2:172.21.15.78:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f531c010df0 con 0x7f5324132370 2024-09-21T09:22:19.476 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.469+0000 7f531a7fc640 1 -- 172.21.15.78:0/103706278 <== mon.0 v2:172.21.15.78:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f531c004b20 con 0x7f5324132370 2024-09-21T09:22:19.476 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.469+0000 7f532bc9a640 1 -- 172.21.15.78:0/103706278 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f5324130df0 con 0x7f5324132370 2024-09-21T09:22:19.477 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.469+0000 7f21ded9d640 1 --2- 172.21.15.78:0/1241159870 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f21e012b3c0 0x7f21e01c2ca0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:22:19.477 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.469+0000 7f21ddd9b640 1 --2- 172.21.15.78:0/1241159870 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f21e006fd10 0x7f21e01c2760 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-09-21T09:22:19.477 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.469+0000 7f21de59c640 1 --2- 172.21.15.78:0/1241159870 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f21e006f3f0 0x7f21e01c2220 secure :-1 s=READY pgs=32 cs=0 l=1 rev1=1 crypto rx=0x7f21d000e9f0 tx=0x7f21d000edf0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:19.477 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.469+0000 7f21ceffd640 1 -- 172.21.15.78:0/1241159870 <== mon.1 v2:172.21.15.78:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f21d000ccc0 con 0x7f21e006f3f0 2024-09-21T09:22:19.477 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.469+0000 7f21ceffd640 1 -- 172.21.15.78:0/1241159870 <== mon.1 v2:172.21.15.78:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f21d0010430 con 0x7f21e006f3f0 2024-09-21T09:22:19.477 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.469+0000 7f21df59e640 1 -- 172.21.15.78:0/1241159870 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f21e012feb0 con 0x7f21e006f3f0 2024-09-21T09:22:19.477 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.469+0000 7f21ceffd640 1 -- 172.21.15.78:0/1241159870 <== mon.1 v2:172.21.15.78:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f21d0010860 con 0x7f21e006f3f0 2024-09-21T09:22:19.478 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.470+0000 7f531a7fc640 1 -- 172.21.15.78:0/103706278 <== mon.0 v2:172.21.15.78:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f531c00d0d0 con 0x7f5324132370 2024-09-21T09:22:19.478 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.470+0000 7f532bc9a640 1 -- 172.21.15.78:0/103706278 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f53241c11e0 con 0x7f5324132370 2024-09-21T09:22:19.478 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.470+0000 7f21df59e640 1 -- 172.21.15.78:0/1241159870 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7f21e0131190 con 0x7f21e006f3f0 2024-09-21T09:22:19.478 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.471+0000 7f531a7fc640 1 -- 172.21.15.78:0/103706278 <== mon.0 v2:172.21.15.78:3300/0 4 ==== mgrmap(e 9) ==== 63442+0+0 (secure 0 0 0) 0x7f531c01f020 con 0x7f5324132370 2024-09-21T09:22:19.479 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705683 got 38654705684 for osd.0 2024-09-21T09:22:19.479 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.471+0000 7f531a7fc640 1 -- 172.21.15.78:0/103706278 <== mon.0 v2:172.21.15.78:3300/0 5 ==== osd_map(20..20 src has 1..20) ==== 3027+0+0 (secure 0 0 0) 0x7f531c072060 con 0x7f5324132370 2024-09-21T09:22:19.480 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.471+0000 7f21ceffd640 1 -- 172.21.15.78:0/1241159870 <== mon.1 v2:172.21.15.78:3301/0 4 ==== mgrmap(e 9) ==== 63442+0+0 (secure 0 0 0) 0x7f21d0010a00 con 0x7f21e006f3f0 2024-09-21T09:22:19.480 DEBUG:teuthology.parallel:result is None 2024-09-21T09:22:19.480 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.472+0000 7f532bc9a640 1 -- 172.21.15.78:0/103706278 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f532409f2e0 con 0x7f5324132370 2024-09-21T09:22:19.480 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.472+0000 7f21df59e640 1 -- 172.21.15.78:0/1241159870 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f21e006f7d0 con 0x7f21e006f3f0 2024-09-21T09:22:19.480 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.472+0000 7f21ceffd640 1 -- 172.21.15.78:0/1241159870 <== mon.1 v2:172.21.15.78:3301/0 5 ==== osd_map(20..20 src has 1..20) ==== 3027+0+0 (secure 0 0 0) 0x7f21d0069a20 con 0x7f21e006f3f0 2024-09-21T09:22:19.481 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.476+0000 7f21ceffd640 1 -- 172.21.15.78:0/1241159870 <== mon.1 v2:172.21.15.78:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f21d0068ac0 con 0x7f21e006f3f0 2024-09-21T09:22:19.481 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.476+0000 7f531a7fc640 1 -- 172.21.15.78:0/103706278 <== mon.0 v2:172.21.15.78:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f531c014040 con 0x7f5324132370 2024-09-21T09:22:19.621 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.619+0000 7fcf9538a640 1 -- 172.21.15.78:0/1324123725 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 2} v 0) -- 0x7fcf90073c10 con 0x7fcf90072950 2024-09-21T09:22:19.622 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.619+0000 7fcf777fe640 1 -- 172.21.15.78:0/1324123725 <== mon.0 v2:172.21.15.78:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 2}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7fcf90073c10 con 0x7fcf90072950 2024-09-21T09:22:19.622 INFO:teuthology.orchestra.run.smithi078.stdout:38654705686 2024-09-21T09:22:19.624 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.621+0000 7fcf9538a640 1 -- 172.21.15.78:0/1324123725 >> v1:172.21.15.78:6816/2186157269 conn(0x7fcf7c051450 legacy=0x7fcf7c053910 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:19.624 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.621+0000 7fcf9538a640 1 -- 172.21.15.78:0/1324123725 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fcf90072950 msgr2=0x7fcf9012d3b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:19.624 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.621+0000 7fcf9538a640 1 --2- 172.21.15.78:0/1324123725 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fcf90072950 0x7fcf9012d3b0 secure :-1 s=READY pgs=61 cs=0 l=1 rev1=1 crypto rx=0x7fcf90073950 tx=0x7fcf84003820 comp rx=0 tx=0).stop 2024-09-21T09:22:19.624 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.622+0000 7fcf9538a640 1 -- 172.21.15.78:0/1324123725 shutdown_connections 2024-09-21T09:22:19.624 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.622+0000 7fcf9538a640 1 --2- 172.21.15.78:0/1324123725 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fcf90097030 0x7fcf9008ffb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:19.624 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.622+0000 7fcf9538a640 1 --2- 172.21.15.78:0/1324123725 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fcf90072950 0x7fcf9012d3b0 unknown :-1 s=CLOSED pgs=61 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:19.624 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.622+0000 7fcf9538a640 1 --2- 172.21.15.78:0/1324123725 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fcf9006f4c0 0x7fcf9012ce70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:19.625 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.622+0000 7fcf9538a640 1 -- 172.21.15.78:0/1324123725 >> 172.21.15.78:0/1324123725 conn(0x7fcf9009dec0 msgr2=0x7fcf90130ff0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:19.625 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.622+0000 7fcf9538a640 1 -- 172.21.15.78:0/1324123725 shutdown_connections 2024-09-21T09:22:19.625 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.622+0000 7fcf9538a640 1 -- 172.21.15.78:0/1324123725 wait complete. 2024-09-21T09:22:19.634 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.632+0000 7f21df59e640 1 -- 172.21.15.78:0/1241159870 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 3} v 0) -- 0x7f21e0130ed0 con 0x7f21e006f3f0 2024-09-21T09:22:19.635 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705685 got 38654705686 for osd.2 2024-09-21T09:22:19.635 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.632+0000 7f21ceffd640 1 -- 172.21.15.78:0/1241159870 <== mon.1 v2:172.21.15.78:3301/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 3}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7f21d0010cf0 con 0x7f21e006f3f0 2024-09-21T09:22:19.635 INFO:teuthology.orchestra.run.smithi078.stdout:38654705684 2024-09-21T09:22:19.635 DEBUG:teuthology.parallel:result is None 2024-09-21T09:22:19.636 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.634+0000 7f21df59e640 1 -- 172.21.15.78:0/1241159870 >> v1:172.21.15.78:6816/2186157269 conn(0x7f21c0051420 legacy=0x7f21c00538e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:19.636 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.634+0000 7f21df59e640 1 -- 172.21.15.78:0/1241159870 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f21e006f3f0 msgr2=0x7f21e01c2220 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:19.636 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.634+0000 7f21df59e640 1 --2- 172.21.15.78:0/1241159870 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f21e006f3f0 0x7f21e01c2220 secure :-1 s=READY pgs=32 cs=0 l=1 rev1=1 crypto rx=0x7f21d000e9f0 tx=0x7f21d000edf0 comp rx=0 tx=0).stop 2024-09-21T09:22:19.637 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.634+0000 7f21df59e640 1 -- 172.21.15.78:0/1241159870 shutdown_connections 2024-09-21T09:22:19.637 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.634+0000 7f21df59e640 1 --2- 172.21.15.78:0/1241159870 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f21e012b3c0 0x7f21e01c2ca0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:19.637 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.634+0000 7f21df59e640 1 --2- 172.21.15.78:0/1241159870 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f21e006fd10 0x7f21e01c2760 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:19.637 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.634+0000 7f21df59e640 1 --2- 172.21.15.78:0/1241159870 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f21e006f3f0 0x7f21e01c2220 unknown :-1 s=CLOSED pgs=32 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:19.637 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.634+0000 7f21df59e640 1 -- 172.21.15.78:0/1241159870 >> 172.21.15.78:0/1241159870 conn(0x7f21e0098530 msgr2=0x7f21e0071670 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:19.637 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.634+0000 7f21df59e640 1 -- 172.21.15.78:0/1241159870 shutdown_connections 2024-09-21T09:22:19.637 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.634+0000 7f21df59e640 1 -- 172.21.15.78:0/1241159870 wait complete. 2024-09-21T09:22:19.647 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705683 got 38654705684 for osd.3 2024-09-21T09:22:19.647 DEBUG:teuthology.parallel:result is None 2024-09-21T09:22:19.656 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.654+0000 7f532bc9a640 1 -- 172.21.15.78:0/103706278 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 1} v 0) -- 0x7f53240699c0 con 0x7f5324132370 2024-09-21T09:22:19.656 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.654+0000 7f531a7fc640 1 -- 172.21.15.78:0/103706278 <== mon.0 v2:172.21.15.78:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 1}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7f531c06b5f0 con 0x7f5324132370 2024-09-21T09:22:19.657 INFO:teuthology.orchestra.run.smithi078.stdout:38654705684 2024-09-21T09:22:19.659 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.656+0000 7f532bc9a640 1 -- 172.21.15.78:0/103706278 >> v1:172.21.15.78:6816/2186157269 conn(0x7f5308051420 legacy=0x7f53080538e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:19.659 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.656+0000 7f532bc9a640 1 -- 172.21.15.78:0/103706278 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f5324132370 msgr2=0x7f532412de10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:19.659 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.656+0000 7f532bc9a640 1 --2- 172.21.15.78:0/103706278 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f5324132370 0x7f532412de10 secure :-1 s=READY pgs=64 cs=0 l=1 rev1=1 crypto rx=0x7f531c012f90 tx=0x7f531c010490 comp rx=0 tx=0).stop 2024-09-21T09:22:19.659 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.656+0000 7f532bc9a640 1 -- 172.21.15.78:0/103706278 shutdown_connections 2024-09-21T09:22:19.659 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.656+0000 7f532bc9a640 1 --2- 172.21.15.78:0/103706278 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f5324132370 0x7f532412de10 unknown :-1 s=CLOSED pgs=64 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:19.659 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.656+0000 7f532bc9a640 1 --2- 172.21.15.78:0/103706278 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f5324123da0 0x7f53241c0ca0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:19.659 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.656+0000 7f532bc9a640 1 --2- 172.21.15.78:0/103706278 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f53241232b0 0x7f53241c0760 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:19.659 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.656+0000 7f532bc9a640 1 -- 172.21.15.78:0/103706278 >> 172.21.15.78:0/103706278 conn(0x7f532409e610 msgr2=0x7f53240994a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:19.659 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.656+0000 7f532bc9a640 1 -- 172.21.15.78:0/103706278 shutdown_connections 2024-09-21T09:22:19.660 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.656+0000 7f532bc9a640 1 -- 172.21.15.78:0/103706278 wait complete. 2024-09-21T09:22:19.669 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705683 got 38654705684 for osd.1 2024-09-21T09:22:19.669 DEBUG:teuthology.parallel:result is None 2024-09-21T09:22:19.669 INFO:tasks.ceph.ceph_manager.ceph:waiting for clean 2024-09-21T09:22:19.669 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-09-21T09:22:19.808 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.806+0000 7f370bdf6640 1 -- 172.21.15.78:0/1184428113 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f370412eff0 msgr2=0x7f370412f460 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:19.808 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.806+0000 7f370bdf6640 1 --2- 172.21.15.78:0/1184428113 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f370412eff0 0x7f370412f460 secure :-1 s=READY pgs=23 cs=0 l=1 rev1=1 crypto rx=0x7f36fc009830 tx=0x7f36fc02a3f0 comp rx=0 tx=0).stop 2024-09-21T09:22:19.809 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.806+0000 7f370bdf6640 1 -- 172.21.15.78:0/1184428113 shutdown_connections 2024-09-21T09:22:19.809 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.806+0000 7f370bdf6640 1 --2- 172.21.15.78:0/1184428113 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f370412eff0 0x7f370412f460 unknown :-1 s=CLOSED pgs=23 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:19.809 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.806+0000 7f370bdf6640 1 --2- 172.21.15.78:0/1184428113 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f37040999a0 0x7f370412eab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:19.809 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.806+0000 7f370bdf6640 1 --2- 172.21.15.78:0/1184428113 >> v2:172.21.15.78:3300/0 conn(0x7f3704099080 0x7f3704099460 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:19.809 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.806+0000 7f370bdf6640 1 -- 172.21.15.78:0/1184428113 >> 172.21.15.78:0/1184428113 conn(0x7f37040a1800 msgr2=0x7f37040a1c10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:19.809 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.806+0000 7f370bdf6640 1 -- 172.21.15.78:0/1184428113 shutdown_connections 2024-09-21T09:22:19.809 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.806+0000 7f370bdf6640 1 -- 172.21.15.78:0/1184428113 wait complete. 2024-09-21T09:22:19.811 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.806+0000 7f370bdf6640 1 Processor -- start 2024-09-21T09:22:19.811 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.807+0000 7f370bdf6640 1 -- start start 2024-09-21T09:22:19.811 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.807+0000 7f370bdf6640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f3704099080 0x7f37041c4dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:19.811 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.807+0000 7f370bdf6640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f37040999a0 0x7f37041c5300 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:19.811 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.807+0000 7f370bdf6640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f370412eff0 0x7f3704071560 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:19.812 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.807+0000 7f370bdf6640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7f37041366c0 con 0x7f3704099080 2024-09-21T09:22:19.812 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.807+0000 7f370bdf6640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7f3704136540 con 0x7f37040999a0 2024-09-21T09:22:19.812 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.807+0000 7f370bdf6640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7f3704136840 con 0x7f370412eff0 2024-09-21T09:22:19.812 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.807+0000 7f3709b92640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f3704099080 0x7f37041c4dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:19.812 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.807+0000 7f3709391640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f37040999a0 0x7f37041c5300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:19.812 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.807+0000 7f3709b92640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f3704099080 0x7f37041c4dc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3300/0 says I am v2:172.21.15.78:51466/0 (socket says 172.21.15.78:51466) 2024-09-21T09:22:19.812 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.807+0000 7f370a393640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f370412eff0 0x7f3704071560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:19.812 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.807+0000 7f3709b92640 1 -- 172.21.15.78:0/2924326665 learned_addr learned my addr 172.21.15.78:0/2924326665 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:19.812 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.807+0000 7f3709391640 1 -- 172.21.15.78:0/2924326665 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f370412eff0 msgr2=0x7f3704071560 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:19.812 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.807+0000 7f3709391640 1 --2- 172.21.15.78:0/2924326665 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f370412eff0 0x7f3704071560 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:19.812 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.807+0000 7f3709391640 1 -- 172.21.15.78:0/2924326665 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f3704099080 msgr2=0x7f37041c4dc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:19.812 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.807+0000 7f3709391640 1 --2- 172.21.15.78:0/2924326665 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f3704099080 0x7f37041c4dc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:19.812 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.807+0000 7f3709391640 1 -- 172.21.15.78:0/2924326665 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f370412feb0 con 0x7f37040999a0 2024-09-21T09:22:19.812 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.807+0000 7f3709b92640 1 --2- 172.21.15.78:0/2924326665 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f3704099080 0x7f37041c4dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:22:19.812 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.807+0000 7f370a393640 1 --2- 172.21.15.78:0/2924326665 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f370412eff0 0x7f3704071560 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:22:19.813 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.808+0000 7f3709391640 1 --2- 172.21.15.78:0/2924326665 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f37040999a0 0x7f37041c5300 secure :-1 s=READY pgs=33 cs=0 l=1 rev1=1 crypto rx=0x7f36f800e9f0 tx=0x7f36f800eeb0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:19.813 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.808+0000 7f37027fc640 1 -- 172.21.15.78:0/2924326665 <== mon.1 v2:172.21.15.78:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f36f800cd80 con 0x7f37040999a0 2024-09-21T09:22:19.813 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.808+0000 7f37027fc640 1 -- 172.21.15.78:0/2924326665 <== mon.1 v2:172.21.15.78:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f36f8004020 con 0x7f37040999a0 2024-09-21T09:22:19.813 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.808+0000 7f37027fc640 1 -- 172.21.15.78:0/2924326665 <== mon.1 v2:172.21.15.78:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f36f8004450 con 0x7f37040999a0 2024-09-21T09:22:19.813 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.808+0000 7f370bdf6640 1 -- 172.21.15.78:0/2924326665 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f370412fba0 con 0x7f37040999a0 2024-09-21T09:22:19.813 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.808+0000 7f370bdf6640 1 -- 172.21.15.78:0/2924326665 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7f37041c5840 con 0x7f37040999a0 2024-09-21T09:22:19.813 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.809+0000 7f370bdf6640 1 -- 172.21.15.78:0/2924326665 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f3704099770 con 0x7f37040999a0 2024-09-21T09:22:19.813 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.809+0000 7f37027fc640 1 -- 172.21.15.78:0/2924326665 <== mon.1 v2:172.21.15.78:3301/0 4 ==== mgrmap(e 9) ==== 63442+0+0 (secure 0 0 0) 0x7f36f80035f0 con 0x7f37040999a0 2024-09-21T09:22:19.813 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.809+0000 7f37027fc640 1 -- 172.21.15.78:0/2924326665 <== mon.1 v2:172.21.15.78:3301/0 5 ==== osd_map(20..20 src has 1..20) ==== 3027+0+0 (secure 0 0 0) 0x7f36f8069750 con 0x7f37040999a0 2024-09-21T09:22:19.815 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.813+0000 7f37027fc640 1 -- 172.21.15.78:0/2924326665 <== mon.1 v2:172.21.15.78:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f3704099770 con 0x7f37040999a0 2024-09-21T09:22:19.962 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.960+0000 7f370bdf6640 1 -- 172.21.15.78:0/2924326665 --> v1:172.21.15.78:6816/2186157269 -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) -- 0x7f37040698c0 con 0x7f36f0051420 2024-09-21T09:22:19.963 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.961+0000 7f37027fc640 1 -- 172.21.15.78:0/2924326665 <== mgr.14103 v1:172.21.15.78:6816/2186157269 1 ==== mgr_command_reply(tid 0: 0 dumped all) ==== 18+0+31971 (unknown 2965378022 0 3990790906) 0x7f37040698c0 con 0x7f36f0051420 2024-09-21T09:22:19.963 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:22:19.964 INFO:teuthology.orchestra.run.smithi078.stderr:dumped all 2024-09-21T09:22:19.965 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.962+0000 7f370bdf6640 1 -- 172.21.15.78:0/2924326665 >> v1:172.21.15.78:6816/2186157269 conn(0x7f36f0051420 legacy=0x7f36f00538e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:19.965 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.962+0000 7f370bdf6640 1 -- 172.21.15.78:0/2924326665 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f37040999a0 msgr2=0x7f37041c5300 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:19.965 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.962+0000 7f370bdf6640 1 --2- 172.21.15.78:0/2924326665 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f37040999a0 0x7f37041c5300 secure :-1 s=READY pgs=33 cs=0 l=1 rev1=1 crypto rx=0x7f36f800e9f0 tx=0x7f36f800eeb0 comp rx=0 tx=0).stop 2024-09-21T09:22:19.965 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.962+0000 7f370bdf6640 1 -- 172.21.15.78:0/2924326665 shutdown_connections 2024-09-21T09:22:19.965 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.962+0000 7f370bdf6640 1 --2- 172.21.15.78:0/2924326665 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f370412eff0 0x7f3704071560 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:19.966 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.962+0000 7f370bdf6640 1 --2- 172.21.15.78:0/2924326665 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f37040999a0 0x7f37041c5300 unknown :-1 s=CLOSED pgs=33 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:19.966 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.962+0000 7f370bdf6640 1 --2- 172.21.15.78:0/2924326665 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f3704099080 0x7f37041c4dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:19.966 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.963+0000 7f370bdf6640 1 -- 172.21.15.78:0/2924326665 >> 172.21.15.78:0/2924326665 conn(0x7f37040a1800 msgr2=0x7f370412bff0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:19.966 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.963+0000 7f370bdf6640 1 -- 172.21.15.78:0/2924326665 shutdown_connections 2024-09-21T09:22:19.966 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:19.963+0000 7f370bdf6640 1 -- 172.21.15.78:0/2924326665 wait complete. 2024-09-21T09:22:19.975 INFO:teuthology.orchestra.run.smithi078.stdout:{"pg_ready":true,"pg_map":{"version":4,"stamp":"2024-09-21T09:22:18.398927+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":139,"num_write_kb":1896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100,"ondisk_log_size":100,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":377487360,"kb_used":109912,"kb_used_data":1832,"kb_used_omap":6,"kb_used_meta":108025,"kb_avail":377377448,"statfs":{"total":386547056640,"available":386434506752,"internally_reserved":0,"allocated":1875968,"data_stored":1380376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6374,"internal_metadata":110618394},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.997025"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":19,"state":"active+clean","last_fresh":"2024-09-21T09:21:04.504065+0000","last_change":"2024-09-21T09:20:58.335685+0000","last_active":"2024-09-21T09:21:04.504065+0000","last_peered":"2024-09-21T09:21:04.504065+0000","last_clean":"2024-09-21T09:21:04.504065+0000","last_became_active":"2024-09-21T09:20:56.322152+0000","last_became_peered":"2024-09-21T09:20:56.322152+0000","last_unstale":"2024-09-21T09:21:04.504065+0000","last_undegraded":"2024-09-21T09:21:04.504065+0000","last_fullsized":"2024-09-21T09:21:04.504065+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:19:22.863125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039605,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":19,"reported_epoch":19,"state":"active+clean","last_fresh":"2024-09-21T09:21:04.775325+0000","last_change":"2024-09-21T09:20:58.335404+0000","last_active":"2024-09-21T09:21:04.775325+0000","last_peered":"2024-09-21T09:21:04.775325+0000","last_clean":"2024-09-21T09:21:04.775325+0000","last_became_active":"2024-09-21T09:20:56.320383+0000","last_became_peered":"2024-09-21T09:20:56.320383+0000","last_unstale":"2024-09-21T09:21:04.775325+0000","last_undegraded":"2024-09-21T09:21:04.775325+0000","last_fullsized":"2024-09-21T09:21:04.775325+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:36:03.357395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040851400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":19,"reported_epoch":19,"state":"active+clean","last_fresh":"2024-09-21T09:21:04.504056+0000","last_change":"2024-09-21T09:20:58.335829+0000","last_active":"2024-09-21T09:21:04.504056+0000","last_peered":"2024-09-21T09:21:04.504056+0000","last_clean":"2024-09-21T09:21:04.504056+0000","last_became_active":"2024-09-21T09:20:56.322789+0000","last_became_peered":"2024-09-21T09:20:56.322789+0000","last_unstale":"2024-09-21T09:21:04.504056+0000","last_undegraded":"2024-09-21T09:21:04.504056+0000","last_fullsized":"2024-09-21T09:21:04.504056+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:16:32.684631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001005517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":19,"reported_epoch":19,"state":"active+clean","last_fresh":"2024-09-21T09:21:04.775318+0000","last_change":"2024-09-21T09:20:58.335406+0000","last_active":"2024-09-21T09:21:04.775318+0000","last_peered":"2024-09-21T09:21:04.775318+0000","last_clean":"2024-09-21T09:21:04.775318+0000","last_became_active":"2024-09-21T09:20:56.324636+0000","last_became_peered":"2024-09-21T09:20:56.324636+0000","last_unstale":"2024-09-21T09:21:04.775318+0000","last_undegraded":"2024-09-21T09:21:04.775318+0000","last_fullsized":"2024-09-21T09:21:04.775318+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:19:22.863125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054560899999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"17'2","reported_seq":21,"reported_epoch":19,"state":"active+clean","last_fresh":"2024-09-21T09:21:04.504212+0000","last_change":"2024-09-21T09:20:58.336768+0000","last_active":"2024-09-21T09:21:04.504212+0000","last_peered":"2024-09-21T09:21:04.504212+0000","last_clean":"2024-09-21T09:21:04.504212+0000","last_became_active":"2024-09-21T09:20:56.327106+0000","last_became_peered":"2024-09-21T09:20:56.327106+0000","last_unstale":"2024-09-21T09:21:04.504212+0000","last_undegraded":"2024-09-21T09:21:04.504212+0000","last_fullsized":"2024-09-21T09:21:04.504212+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:45:36.558638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011881540000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":19,"reported_epoch":19,"state":"active+clean","last_fresh":"2024-09-21T09:21:05.946921+0000","last_change":"2024-09-21T09:20:58.479452+0000","last_active":"2024-09-21T09:21:05.946921+0000","last_peered":"2024-09-21T09:21:05.946921+0000","last_clean":"2024-09-21T09:21:05.946921+0000","last_became_active":"2024-09-21T09:20:56.324347+0000","last_became_peered":"2024-09-21T09:20:56.324347+0000","last_unstale":"2024-09-21T09:21:05.946921+0000","last_undegraded":"2024-09-21T09:21:05.946921+0000","last_fullsized":"2024-09-21T09:21:05.946921+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:43:56.319711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023051899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":19,"reported_epoch":19,"state":"active+clean","last_fresh":"2024-09-21T09:21:04.504106+0000","last_change":"2024-09-21T09:20:58.335174+0000","last_active":"2024-09-21T09:21:04.504106+0000","last_peered":"2024-09-21T09:21:04.504106+0000","last_clean":"2024-09-21T09:21:04.504106+0000","last_became_active":"2024-09-21T09:20:56.322710+0000","last_became_peered":"2024-09-21T09:20:56.322710+0000","last_unstale":"2024-09-21T09:21:04.504106+0000","last_undegraded":"2024-09-21T09:21:04.504106+0000","last_fullsized":"2024-09-21T09:21:04.504106+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:03:03.617340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039429200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"15'1","reported_seq":20,"reported_epoch":19,"state":"active+clean","last_fresh":"2024-09-21T09:21:04.504015+0000","last_change":"2024-09-21T09:20:58.336756+0000","last_active":"2024-09-21T09:21:04.504015+0000","last_peered":"2024-09-21T09:21:04.504015+0000","last_clean":"2024-09-21T09:21:04.504015+0000","last_became_active":"2024-09-21T09:20:56.321532+0000","last_became_peered":"2024-09-21T09:20:56.321532+0000","last_unstale":"2024-09-21T09:21:04.504015+0000","last_undegraded":"2024-09-21T09:21:04.504015+0000","last_fullsized":"2024-09-21T09:21:04.504015+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:57:55.907381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091827900000000004,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"12'97","reported_seq":144,"reported_epoch":19,"state":"active+clean","last_fresh":"2024-09-21T09:21:04.775317+0000","last_change":"2024-09-21T09:20:52.294610+0000","last_active":"2024-09-21T09:21:04.775317+0000","last_peered":"2024-09-21T09:21:04.775317+0000","last_clean":"2024-09-21T09:21:04.775317+0000","last_became_active":"2024-09-21T09:20:52.293626+0000","last_became_peered":"2024-09-21T09:20:52.293626+0000","last_unstale":"2024-09-21T09:21:04.775317+0000","last_undegraded":"2024-09-21T09:21:04.775317+0000","last_fullsized":"2024-09-21T09:21:04.775317+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:51.278939+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:51.278939+0000","last_clean_scrub_stamp":"2024-09-21T09:20:51.278939+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:43:02.095040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":4},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97,"ondisk_log_size":97,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":3,"up_from":9,"seq":38654705684,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27192,"kb_used_data":172,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344648,"statfs":{"total":96636764160,"available":96608919552,"internally_reserved":0,"allocated":176128,"data_stored":50780,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":27654600},"hb_peers":[0,1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":9,"seq":38654705686,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27188,"kb_used_data":168,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344652,"statfs":{"total":96636764160,"available":96608923648,"internally_reserved":0,"allocated":172032,"data_stored":50761,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":9,"seq":38654705684,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27764,"kb_used_data":744,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344076,"statfs":{"total":96636764160,"available":96608333824,"internally_reserved":0,"allocated":761856,"data_stored":637706,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":9,"seq":38654705684,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27768,"kb_used_data":748,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344072,"statfs":{"total":96636764160,"available":96608329728,"internally_reserved":0,"allocated":765952,"data_stored":641129,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[1,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T09:22:19.976 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-09-21T09:22:20.079 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.073+0000 7f4fa6dc0640 1 -- 172.21.15.78:0/3032484931 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f4fa00916e0 msgr2=0x7f4fa0134ea0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:20.080 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.073+0000 7f4fa6dc0640 1 --2- 172.21.15.78:0/3032484931 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f4fa00916e0 0x7f4fa0134ea0 secure :-1 s=READY pgs=24 cs=0 l=1 rev1=1 crypto rx=0x7f4f98009be0 tx=0x7f4f9802a770 comp rx=0 tx=0).stop 2024-09-21T09:22:20.080 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.073+0000 7f4fa6dc0640 1 -- 172.21.15.78:0/3032484931 shutdown_connections 2024-09-21T09:22:20.080 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.073+0000 7f4fa6dc0640 1 --2- 172.21.15.78:0/3032484931 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f4fa00916e0 0x7f4fa0134ea0 unknown :-1 s=CLOSED pgs=24 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:20.080 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.073+0000 7f4fa6dc0640 1 --2- 172.21.15.78:0/3032484931 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f4fa006e2d0 0x7f4fa006e6b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:20.080 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.073+0000 7f4fa6dc0640 1 -- 172.21.15.78:0/3032484931 >> 172.21.15.78:0/3032484931 conn(0x7f4fa009c230 msgr2=0x7f4fa009c640 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:20.080 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.073+0000 7f4fa6dc0640 1 -- 172.21.15.78:0/3032484931 shutdown_connections 2024-09-21T09:22:20.080 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.073+0000 7f4fa6dc0640 1 -- 172.21.15.78:0/3032484931 wait complete. 2024-09-21T09:22:20.080 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.074+0000 7f4fa6dc0640 1 Processor -- start 2024-09-21T09:22:20.080 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.074+0000 7f4fa6dc0640 1 -- start start 2024-09-21T09:22:20.080 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.075+0000 7f4fa6dc0640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f4fa006e2d0 0x7f4fa006f880 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:20.081 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.075+0000 7f4fa6dc0640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f4fa006ebf0 0x7f4fa006fdc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:20.081 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.075+0000 7f4fa6dc0640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f4fa00916e0 0x7f4fa0070300 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:20.081 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.075+0000 7f4fa6dc0640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7f4fa01358b0 con 0x7f4fa006e2d0 2024-09-21T09:22:20.081 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.075+0000 7f4fa6dc0640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7f4fa0135730 con 0x7f4fa006ebf0 2024-09-21T09:22:20.081 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.075+0000 7f4fa6dc0640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7f4fa0135a30 con 0x7f4fa00916e0 2024-09-21T09:22:20.081 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.075+0000 7f4fa4b5c640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f4fa006e2d0 0x7f4fa006f880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:20.081 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.075+0000 7f4fa535d640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f4fa00916e0 0x7f4fa0070300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:20.081 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.075+0000 7f4fa4b5c640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f4fa006e2d0 0x7f4fa006f880 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3300/0 says I am v2:172.21.15.78:51480/0 (socket says 172.21.15.78:51480) 2024-09-21T09:22:20.081 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.075+0000 7f4f9ffff640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f4fa006ebf0 0x7f4fa006fdc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:20.081 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.075+0000 7f4fa4b5c640 1 -- 172.21.15.78:0/156813853 learned_addr learned my addr 172.21.15.78:0/156813853 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:20.081 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.075+0000 7f4fa4b5c640 1 -- 172.21.15.78:0/156813853 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f4fa00916e0 msgr2=0x7f4fa0070300 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:20.081 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.075+0000 7f4fa4b5c640 1 --2- 172.21.15.78:0/156813853 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f4fa00916e0 0x7f4fa0070300 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:20.081 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.075+0000 7f4fa4b5c640 1 -- 172.21.15.78:0/156813853 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f4fa006ebf0 msgr2=0x7f4fa006fdc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:20.082 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.075+0000 7f4fa4b5c640 1 --2- 172.21.15.78:0/156813853 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f4fa006ebf0 0x7f4fa006fdc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:20.082 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.075+0000 7f4fa4b5c640 1 -- 172.21.15.78:0/156813853 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f4fa00718e0 con 0x7f4fa006e2d0 2024-09-21T09:22:20.082 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.075+0000 7f4fa535d640 1 --2- 172.21.15.78:0/156813853 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f4fa00916e0 0x7f4fa0070300 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:22:20.082 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.075+0000 7f4fa4b5c640 1 --2- 172.21.15.78:0/156813853 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f4fa006e2d0 0x7f4fa006f880 secure :-1 s=READY pgs=66 cs=0 l=1 rev1=1 crypto rx=0x7f4f9400ea60 tx=0x7f4f9400ef20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:20.082 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.075+0000 7f4f9d7fa640 1 -- 172.21.15.78:0/156813853 <== mon.0 v2:172.21.15.78:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4f9400cde0 con 0x7f4fa006e2d0 2024-09-21T09:22:20.082 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.075+0000 7f4f9d7fa640 1 -- 172.21.15.78:0/156813853 <== mon.0 v2:172.21.15.78:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f4f94004020 con 0x7f4fa006e2d0 2024-09-21T09:22:20.082 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.075+0000 7f4fa6dc0640 1 -- 172.21.15.78:0/156813853 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f4fa01271c0 con 0x7f4fa006e2d0 2024-09-21T09:22:20.082 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.075+0000 7f4f9d7fa640 1 -- 172.21.15.78:0/156813853 <== mon.0 v2:172.21.15.78:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4f94004450 con 0x7f4fa006e2d0 2024-09-21T09:22:20.082 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.075+0000 7f4fa6dc0640 1 -- 172.21.15.78:0/156813853 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f4fa0126060 con 0x7f4fa006e2d0 2024-09-21T09:22:20.082 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.076+0000 7f4fa6dc0640 1 -- 172.21.15.78:0/156813853 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f4fa006e6b0 con 0x7f4fa006e2d0 2024-09-21T09:22:20.082 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.076+0000 7f4f9d7fa640 1 -- 172.21.15.78:0/156813853 <== mon.0 v2:172.21.15.78:3300/0 4 ==== mgrmap(e 9) ==== 63442+0+0 (secure 0 0 0) 0x7f4f940035f0 con 0x7f4fa006e2d0 2024-09-21T09:22:20.082 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.077+0000 7f4f9d7fa640 1 -- 172.21.15.78:0/156813853 <== mon.0 v2:172.21.15.78:3300/0 5 ==== osd_map(20..20 src has 1..20) ==== 3027+0+0 (secure 0 0 0) 0x7f4f940698f0 con 0x7f4fa006e2d0 2024-09-21T09:22:20.083 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.080+0000 7f4f9d7fa640 1 -- 172.21.15.78:0/156813853 <== mon.0 v2:172.21.15.78:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f4f94068550 con 0x7f4fa006e2d0 2024-09-21T09:22:20.228 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.225+0000 7f4fa6dc0640 1 -- 172.21.15.78:0/156813853 --> v1:172.21.15.78:6816/2186157269 -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) -- 0x7f4fa0069950 con 0x7f4f84051420 2024-09-21T09:22:20.228 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.226+0000 7f4f9d7fa640 1 -- 172.21.15.78:0/156813853 <== mgr.14103 v1:172.21.15.78:6816/2186157269 1 ==== mgr_command_reply(tid 0: 0 dumped all) ==== 18+0+31971 (unknown 2965378022 0 3990790906) 0x7f4fa0069950 con 0x7f4f84051420 2024-09-21T09:22:20.229 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:22:20.229 INFO:teuthology.orchestra.run.smithi078.stderr:dumped all 2024-09-21T09:22:20.230 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.228+0000 7f4fa6dc0640 1 -- 172.21.15.78:0/156813853 >> v1:172.21.15.78:6816/2186157269 conn(0x7f4f84051420 legacy=0x7f4f840538e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:20.231 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.228+0000 7f4fa6dc0640 1 -- 172.21.15.78:0/156813853 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f4fa006e2d0 msgr2=0x7f4fa006f880 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:20.231 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.228+0000 7f4fa6dc0640 1 --2- 172.21.15.78:0/156813853 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f4fa006e2d0 0x7f4fa006f880 secure :-1 s=READY pgs=66 cs=0 l=1 rev1=1 crypto rx=0x7f4f9400ea60 tx=0x7f4f9400ef20 comp rx=0 tx=0).stop 2024-09-21T09:22:20.231 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.228+0000 7f4fa6dc0640 1 -- 172.21.15.78:0/156813853 shutdown_connections 2024-09-21T09:22:20.231 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.228+0000 7f4fa6dc0640 1 --2- 172.21.15.78:0/156813853 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f4fa00916e0 0x7f4fa0070300 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:20.231 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.228+0000 7f4fa6dc0640 1 --2- 172.21.15.78:0/156813853 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f4fa006ebf0 0x7f4fa006fdc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:20.231 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.228+0000 7f4fa6dc0640 1 --2- 172.21.15.78:0/156813853 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f4fa006e2d0 0x7f4fa006f880 unknown :-1 s=CLOSED pgs=66 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:20.231 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.228+0000 7f4fa6dc0640 1 -- 172.21.15.78:0/156813853 >> 172.21.15.78:0/156813853 conn(0x7f4fa009c230 msgr2=0x7f4fa012c720 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:20.231 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.228+0000 7f4fa6dc0640 1 -- 172.21.15.78:0/156813853 shutdown_connections 2024-09-21T09:22:20.231 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.228+0000 7f4fa6dc0640 1 -- 172.21.15.78:0/156813853 wait complete. 2024-09-21T09:22:20.241 INFO:teuthology.orchestra.run.smithi078.stdout:{"pg_ready":true,"pg_map":{"version":4,"stamp":"2024-09-21T09:22:18.398927+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":139,"num_write_kb":1896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100,"ondisk_log_size":100,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":377487360,"kb_used":109912,"kb_used_data":1832,"kb_used_omap":6,"kb_used_meta":108025,"kb_avail":377377448,"statfs":{"total":386547056640,"available":386434506752,"internally_reserved":0,"allocated":1875968,"data_stored":1380376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6374,"internal_metadata":110618394},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.997025"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":19,"state":"active+clean","last_fresh":"2024-09-21T09:21:04.504065+0000","last_change":"2024-09-21T09:20:58.335685+0000","last_active":"2024-09-21T09:21:04.504065+0000","last_peered":"2024-09-21T09:21:04.504065+0000","last_clean":"2024-09-21T09:21:04.504065+0000","last_became_active":"2024-09-21T09:20:56.322152+0000","last_became_peered":"2024-09-21T09:20:56.322152+0000","last_unstale":"2024-09-21T09:21:04.504065+0000","last_undegraded":"2024-09-21T09:21:04.504065+0000","last_fullsized":"2024-09-21T09:21:04.504065+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:19:22.863125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039605,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":19,"reported_epoch":19,"state":"active+clean","last_fresh":"2024-09-21T09:21:04.775325+0000","last_change":"2024-09-21T09:20:58.335404+0000","last_active":"2024-09-21T09:21:04.775325+0000","last_peered":"2024-09-21T09:21:04.775325+0000","last_clean":"2024-09-21T09:21:04.775325+0000","last_became_active":"2024-09-21T09:20:56.320383+0000","last_became_peered":"2024-09-21T09:20:56.320383+0000","last_unstale":"2024-09-21T09:21:04.775325+0000","last_undegraded":"2024-09-21T09:21:04.775325+0000","last_fullsized":"2024-09-21T09:21:04.775325+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:36:03.357395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040851400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":19,"reported_epoch":19,"state":"active+clean","last_fresh":"2024-09-21T09:21:04.504056+0000","last_change":"2024-09-21T09:20:58.335829+0000","last_active":"2024-09-21T09:21:04.504056+0000","last_peered":"2024-09-21T09:21:04.504056+0000","last_clean":"2024-09-21T09:21:04.504056+0000","last_became_active":"2024-09-21T09:20:56.322789+0000","last_became_peered":"2024-09-21T09:20:56.322789+0000","last_unstale":"2024-09-21T09:21:04.504056+0000","last_undegraded":"2024-09-21T09:21:04.504056+0000","last_fullsized":"2024-09-21T09:21:04.504056+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:16:32.684631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001005517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":19,"reported_epoch":19,"state":"active+clean","last_fresh":"2024-09-21T09:21:04.775318+0000","last_change":"2024-09-21T09:20:58.335406+0000","last_active":"2024-09-21T09:21:04.775318+0000","last_peered":"2024-09-21T09:21:04.775318+0000","last_clean":"2024-09-21T09:21:04.775318+0000","last_became_active":"2024-09-21T09:20:56.324636+0000","last_became_peered":"2024-09-21T09:20:56.324636+0000","last_unstale":"2024-09-21T09:21:04.775318+0000","last_undegraded":"2024-09-21T09:21:04.775318+0000","last_fullsized":"2024-09-21T09:21:04.775318+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:19:22.863125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054560899999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"17'2","reported_seq":21,"reported_epoch":19,"state":"active+clean","last_fresh":"2024-09-21T09:21:04.504212+0000","last_change":"2024-09-21T09:20:58.336768+0000","last_active":"2024-09-21T09:21:04.504212+0000","last_peered":"2024-09-21T09:21:04.504212+0000","last_clean":"2024-09-21T09:21:04.504212+0000","last_became_active":"2024-09-21T09:20:56.327106+0000","last_became_peered":"2024-09-21T09:20:56.327106+0000","last_unstale":"2024-09-21T09:21:04.504212+0000","last_undegraded":"2024-09-21T09:21:04.504212+0000","last_fullsized":"2024-09-21T09:21:04.504212+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:45:36.558638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011881540000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":19,"reported_epoch":19,"state":"active+clean","last_fresh":"2024-09-21T09:21:05.946921+0000","last_change":"2024-09-21T09:20:58.479452+0000","last_active":"2024-09-21T09:21:05.946921+0000","last_peered":"2024-09-21T09:21:05.946921+0000","last_clean":"2024-09-21T09:21:05.946921+0000","last_became_active":"2024-09-21T09:20:56.324347+0000","last_became_peered":"2024-09-21T09:20:56.324347+0000","last_unstale":"2024-09-21T09:21:05.946921+0000","last_undegraded":"2024-09-21T09:21:05.946921+0000","last_fullsized":"2024-09-21T09:21:05.946921+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:43:56.319711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023051899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":19,"reported_epoch":19,"state":"active+clean","last_fresh":"2024-09-21T09:21:04.504106+0000","last_change":"2024-09-21T09:20:58.335174+0000","last_active":"2024-09-21T09:21:04.504106+0000","last_peered":"2024-09-21T09:21:04.504106+0000","last_clean":"2024-09-21T09:21:04.504106+0000","last_became_active":"2024-09-21T09:20:56.322710+0000","last_became_peered":"2024-09-21T09:20:56.322710+0000","last_unstale":"2024-09-21T09:21:04.504106+0000","last_undegraded":"2024-09-21T09:21:04.504106+0000","last_fullsized":"2024-09-21T09:21:04.504106+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:03:03.617340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039429200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"15'1","reported_seq":20,"reported_epoch":19,"state":"active+clean","last_fresh":"2024-09-21T09:21:04.504015+0000","last_change":"2024-09-21T09:20:58.336756+0000","last_active":"2024-09-21T09:21:04.504015+0000","last_peered":"2024-09-21T09:21:04.504015+0000","last_clean":"2024-09-21T09:21:04.504015+0000","last_became_active":"2024-09-21T09:20:56.321532+0000","last_became_peered":"2024-09-21T09:20:56.321532+0000","last_unstale":"2024-09-21T09:21:04.504015+0000","last_undegraded":"2024-09-21T09:21:04.504015+0000","last_fullsized":"2024-09-21T09:21:04.504015+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:57:55.907381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091827900000000004,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"12'97","reported_seq":144,"reported_epoch":19,"state":"active+clean","last_fresh":"2024-09-21T09:21:04.775317+0000","last_change":"2024-09-21T09:20:52.294610+0000","last_active":"2024-09-21T09:21:04.775317+0000","last_peered":"2024-09-21T09:21:04.775317+0000","last_clean":"2024-09-21T09:21:04.775317+0000","last_became_active":"2024-09-21T09:20:52.293626+0000","last_became_peered":"2024-09-21T09:20:52.293626+0000","last_unstale":"2024-09-21T09:21:04.775317+0000","last_undegraded":"2024-09-21T09:21:04.775317+0000","last_fullsized":"2024-09-21T09:21:04.775317+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:51.278939+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:51.278939+0000","last_clean_scrub_stamp":"2024-09-21T09:20:51.278939+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:43:02.095040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":4},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97,"ondisk_log_size":97,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":3,"up_from":9,"seq":38654705684,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27192,"kb_used_data":172,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344648,"statfs":{"total":96636764160,"available":96608919552,"internally_reserved":0,"allocated":176128,"data_stored":50780,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":27654600},"hb_peers":[0,1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":9,"seq":38654705686,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27188,"kb_used_data":168,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344652,"statfs":{"total":96636764160,"available":96608923648,"internally_reserved":0,"allocated":172032,"data_stored":50761,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":9,"seq":38654705684,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27764,"kb_used_data":744,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344076,"statfs":{"total":96636764160,"available":96608333824,"internally_reserved":0,"allocated":761856,"data_stored":637706,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":9,"seq":38654705684,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27768,"kb_used_data":748,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344072,"statfs":{"total":96636764160,"available":96608329728,"internally_reserved":0,"allocated":765952,"data_stored":641129,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[1,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T09:22:20.242 INFO:tasks.ceph.ceph_manager.ceph:clean! 2024-09-21T09:22:20.242 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-21T09:22:20.242 INFO:tasks.ceph.ceph_manager.ceph:wait_until_healthy 2024-09-21T09:22:20.242 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph health --format=json 2024-09-21T09:22:20.341 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.339+0000 7f09e716a640 1 -- 172.21.15.78:0/2322051330 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f09e012cc30 msgr2=0x7f09e012d010 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:20.341 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.339+0000 7f09e716a640 1 --2- 172.21.15.78:0/2322051330 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f09e012cc30 0x7f09e012d010 secure :-1 s=READY pgs=25 cs=0 l=1 rev1=1 crypto rx=0x7f09d8005fd0 tx=0x7f09d802a4b0 comp rx=0 tx=0).stop 2024-09-21T09:22:20.342 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.339+0000 7f09e716a640 1 -- 172.21.15.78:0/2322051330 shutdown_connections 2024-09-21T09:22:20.342 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.339+0000 7f09e716a640 1 --2- 172.21.15.78:0/2322051330 >> v2:172.21.15.78:3300/0 conn(0x7f09e0095110 0x7f09e0134c10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:20.342 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.339+0000 7f09e716a640 1 --2- 172.21.15.78:0/2322051330 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f09e012cc30 0x7f09e012d010 unknown :-1 s=CLOSED pgs=25 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:20.342 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.339+0000 7f09e716a640 1 -- 172.21.15.78:0/2322051330 >> 172.21.15.78:0/2322051330 conn(0x7f09e009a2e0 msgr2=0x7f09e0097ad0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:20.342 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.339+0000 7f09e716a640 1 -- 172.21.15.78:0/2322051330 shutdown_connections 2024-09-21T09:22:20.342 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.339+0000 7f09e716a640 1 -- 172.21.15.78:0/2322051330 wait complete. 2024-09-21T09:22:20.344 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.340+0000 7f09e716a640 1 Processor -- start 2024-09-21T09:22:20.344 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.340+0000 7f09e716a640 1 -- start start 2024-09-21T09:22:20.344 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.340+0000 7f09e716a640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f09e00945d0 0x7f09e0073910 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:20.344 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.340+0000 7f09e716a640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f09e0095110 0x7f09e0073e50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:20.344 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.340+0000 7f09e716a640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f09e012cc30 0x7f09e00910d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:20.344 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.340+0000 7f09e716a640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7f09e0135820 con 0x7f09e00945d0 2024-09-21T09:22:20.344 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.340+0000 7f09e716a640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7f09e01356a0 con 0x7f09e0095110 2024-09-21T09:22:20.344 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.340+0000 7f09e716a640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7f09e01359a0 con 0x7f09e012cc30 2024-09-21T09:22:20.345 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.340+0000 7f09e4f06640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f09e00945d0 0x7f09e0073910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:20.345 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.340+0000 7f09e4f06640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f09e00945d0 0x7f09e0073910 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3300/0 says I am v2:172.21.15.78:51504/0 (socket says 172.21.15.78:51504) 2024-09-21T09:22:20.345 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.340+0000 7f09e4f06640 1 -- 172.21.15.78:0/3917919616 learned_addr learned my addr 172.21.15.78:0/3917919616 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:20.345 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.340+0000 7f09e5707640 1 --2- 172.21.15.78:0/3917919616 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f09e012cc30 0x7f09e00910d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:20.345 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.340+0000 7f09dffff640 1 --2- 172.21.15.78:0/3917919616 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f09e0095110 0x7f09e0073e50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:20.345 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.341+0000 7f09e4f06640 1 -- 172.21.15.78:0/3917919616 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f09e012cc30 msgr2=0x7f09e00910d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:20.345 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.341+0000 7f09e4f06640 1 --2- 172.21.15.78:0/3917919616 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f09e012cc30 0x7f09e00910d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:20.345 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.341+0000 7f09e4f06640 1 -- 172.21.15.78:0/3917919616 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f09e0095110 msgr2=0x7f09e0073e50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:20.345 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.341+0000 7f09e4f06640 1 --2- 172.21.15.78:0/3917919616 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f09e0095110 0x7f09e0073e50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:20.345 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.341+0000 7f09e4f06640 1 -- 172.21.15.78:0/3917919616 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f09e006ee10 con 0x7f09e00945d0 2024-09-21T09:22:20.345 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.341+0000 7f09e5707640 1 --2- 172.21.15.78:0/3917919616 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f09e012cc30 0x7f09e00910d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:22:20.345 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.341+0000 7f09e4f06640 1 --2- 172.21.15.78:0/3917919616 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f09e00945d0 0x7f09e0073910 secure :-1 s=READY pgs=68 cs=0 l=1 rev1=1 crypto rx=0x7f09d8009850 tx=0x7f09d80038a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:20.345 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.341+0000 7f09dd7fa640 1 -- 172.21.15.78:0/3917919616 <== mon.0 v2:172.21.15.78:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f09d803f070 con 0x7f09e00945d0 2024-09-21T09:22:20.346 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.341+0000 7f09dd7fa640 1 -- 172.21.15.78:0/3917919616 <== mon.0 v2:172.21.15.78:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f09d80047a0 con 0x7f09e00945d0 2024-09-21T09:22:20.346 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.341+0000 7f09e716a640 1 -- 172.21.15.78:0/3917919616 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f09e0092ab0 con 0x7f09e00945d0 2024-09-21T09:22:20.346 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.341+0000 7f09dd7fa640 1 -- 172.21.15.78:0/3917919616 <== mon.0 v2:172.21.15.78:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f09d8035030 con 0x7f09e00945d0 2024-09-21T09:22:20.346 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.341+0000 7f09e716a640 1 -- 172.21.15.78:0/3917919616 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f09e0074390 con 0x7f09e00945d0 2024-09-21T09:22:20.346 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.342+0000 7f09e716a640 1 -- 172.21.15.78:0/3917919616 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f09e012d010 con 0x7f09e00945d0 2024-09-21T09:22:20.346 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.342+0000 7f09dd7fa640 1 -- 172.21.15.78:0/3917919616 <== mon.0 v2:172.21.15.78:3300/0 4 ==== mgrmap(e 9) ==== 63442+0+0 (secure 0 0 0) 0x7f09d8042070 con 0x7f09e00945d0 2024-09-21T09:22:20.347 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.343+0000 7f09dd7fa640 1 -- 172.21.15.78:0/3917919616 <== mon.0 v2:172.21.15.78:3300/0 5 ==== osd_map(20..20 src has 1..20) ==== 3027+0+0 (secure 0 0 0) 0x7f09d808a0c0 con 0x7f09e00945d0 2024-09-21T09:22:20.348 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.346+0000 7f09dd7fa640 1 -- 172.21.15.78:0/3917919616 <== mon.0 v2:172.21.15.78:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f09e0092ab0 con 0x7f09e00945d0 2024-09-21T09:22:20.522 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.520+0000 7f09e716a640 1 -- 172.21.15.78:0/3917919616 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "health", "format": "json"} v 0) -- 0x7f09e00699c0 con 0x7f09e00945d0 2024-09-21T09:22:20.523 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.520+0000 7f09dd7fa640 1 -- 172.21.15.78:0/3917919616 <== mon.0 v2:172.21.15.78:3300/0 7 ==== mon_command_ack([{"prefix": "health", "format": "json"}]=0 v0) ==== 72+0+46 (secure 0 0 0) 0x7f09d8088d20 con 0x7f09e00945d0 2024-09-21T09:22:20.523 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:22:20.523 INFO:teuthology.orchestra.run.smithi078.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-21T09:22:20.525 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.522+0000 7f09e716a640 1 -- 172.21.15.78:0/3917919616 >> v1:172.21.15.78:6816/2186157269 conn(0x7f09c0051420 legacy=0x7f09c00538e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:20.525 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.522+0000 7f09e716a640 1 -- 172.21.15.78:0/3917919616 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f09e00945d0 msgr2=0x7f09e0073910 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:20.525 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.522+0000 7f09e716a640 1 --2- 172.21.15.78:0/3917919616 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f09e00945d0 0x7f09e0073910 secure :-1 s=READY pgs=68 cs=0 l=1 rev1=1 crypto rx=0x7f09d8009850 tx=0x7f09d80038a0 comp rx=0 tx=0).stop 2024-09-21T09:22:20.525 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.522+0000 7f09e716a640 1 -- 172.21.15.78:0/3917919616 shutdown_connections 2024-09-21T09:22:20.525 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.522+0000 7f09e716a640 1 --2- 172.21.15.78:0/3917919616 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f09e012cc30 0x7f09e00910d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:20.525 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.522+0000 7f09e716a640 1 --2- 172.21.15.78:0/3917919616 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f09e0095110 0x7f09e0073e50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:20.525 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.522+0000 7f09e716a640 1 --2- 172.21.15.78:0/3917919616 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f09e00945d0 0x7f09e0073910 unknown :-1 s=CLOSED pgs=68 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:20.525 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.522+0000 7f09e716a640 1 -- 172.21.15.78:0/3917919616 >> 172.21.15.78:0/3917919616 conn(0x7f09e009a2e0 msgr2=0x7f09e0129400 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:20.525 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.522+0000 7f09e716a640 1 -- 172.21.15.78:0/3917919616 shutdown_connections 2024-09-21T09:22:20.525 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.522+0000 7f09e716a640 1 -- 172.21.15.78:0/3917919616 wait complete. 2024-09-21T09:22:20.535 INFO:tasks.ceph.ceph_manager.ceph:wait_until_healthy done 2024-09-21T09:22:20.535 INFO:teuthology.run_tasks:Running task ceph.restart... 2024-09-21T09:22:20.545 DEBUG:tasks.ceph.osd.0:waiting for process to exit 2024-09-21T09:22:20.545 INFO:teuthology.orchestra.run:waiting for 300 2024-09-21T09:22:20.580 INFO:tasks.ceph.osd.0:Stopped 2024-09-21T09:22:20.580 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd down 0 2024-09-21T09:22:20.721 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.718+0000 7f6d8f1c5640 1 -- 172.21.15.78:0/982176702 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f6d8812b490 msgr2=0x7f6d8812b870 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:20.721 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.718+0000 7f6d8f1c5640 1 --2- 172.21.15.78:0/982176702 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f6d8812b490 0x7f6d8812b870 secure :-1 s=READY pgs=26 cs=0 l=1 rev1=1 crypto rx=0x7f6d78009be0 tx=0x7f6d7802a840 comp rx=0 tx=0).stop 2024-09-21T09:22:20.721 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.718+0000 7f6d8f1c5640 1 -- 172.21.15.78:0/982176702 shutdown_connections 2024-09-21T09:22:20.721 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.718+0000 7f6d8f1c5640 1 --2- 172.21.15.78:0/982176702 >> v2:172.21.15.78:3300/0 conn(0x7f6d88095570 0x7f6d880959e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:20.721 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.718+0000 7f6d8f1c5640 1 --2- 172.21.15.78:0/982176702 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f6d8812b490 0x7f6d8812b870 unknown :-1 s=CLOSED pgs=26 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:20.721 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.718+0000 7f6d8f1c5640 1 -- 172.21.15.78:0/982176702 >> 172.21.15.78:0/982176702 conn(0x7f6d8809afd0 msgr2=0x7f6d8809b3e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:20.721 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.718+0000 7f6d8f1c5640 1 -- 172.21.15.78:0/982176702 shutdown_connections 2024-09-21T09:22:20.721 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.718+0000 7f6d8f1c5640 1 -- 172.21.15.78:0/982176702 wait complete. 2024-09-21T09:22:20.723 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.719+0000 7f6d8f1c5640 1 Processor -- start 2024-09-21T09:22:20.723 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.719+0000 7f6d8f1c5640 1 -- start start 2024-09-21T09:22:20.723 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.719+0000 7f6d8f1c5640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f6d88095570 0x7f6d88134490 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:20.723 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.719+0000 7f6d8f1c5640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f6d8812b490 0x7f6d881349d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:20.724 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.719+0000 7f6d8f1c5640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f6d8812bdb0 0x7f6d88134f10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:20.724 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.719+0000 7f6d8f1c5640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7f6d88133600 con 0x7f6d88095570 2024-09-21T09:22:20.724 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.719+0000 7f6d8f1c5640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7f6d88133480 con 0x7f6d8812bdb0 2024-09-21T09:22:20.724 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.719+0000 7f6d8f1c5640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7f6d88133780 con 0x7f6d8812b490 2024-09-21T09:22:20.724 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.719+0000 7f6d8cf61640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f6d88095570 0x7f6d88134490 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:20.724 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.719+0000 7f6d87fff640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f6d8812b490 0x7f6d881349d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:20.724 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.719+0000 7f6d8d762640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f6d8812bdb0 0x7f6d88134f10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:20.724 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.719+0000 7f6d8cf61640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f6d88095570 0x7f6d88134490 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3300/0 says I am v2:172.21.15.78:51526/0 (socket says 172.21.15.78:51526) 2024-09-21T09:22:20.724 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.719+0000 7f6d8cf61640 1 -- 172.21.15.78:0/1217016011 learned_addr learned my addr 172.21.15.78:0/1217016011 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:20.724 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.720+0000 7f6d8d762640 1 -- 172.21.15.78:0/1217016011 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f6d8812b490 msgr2=0x7f6d881349d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:20.724 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.720+0000 7f6d8d762640 1 --2- 172.21.15.78:0/1217016011 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f6d8812b490 0x7f6d881349d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:20.724 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.720+0000 7f6d8d762640 1 -- 172.21.15.78:0/1217016011 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f6d88095570 msgr2=0x7f6d88134490 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:20.724 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.720+0000 7f6d8d762640 1 --2- 172.21.15.78:0/1217016011 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f6d88095570 0x7f6d88134490 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:20.725 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.720+0000 7f6d8d762640 1 -- 172.21.15.78:0/1217016011 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f6d88073ef0 con 0x7f6d8812bdb0 2024-09-21T09:22:20.725 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.720+0000 7f6d87fff640 1 --2- 172.21.15.78:0/1217016011 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f6d8812b490 0x7f6d881349d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:22:20.725 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.720+0000 7f6d8d762640 1 --2- 172.21.15.78:0/1217016011 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f6d8812bdb0 0x7f6d88134f10 secure :-1 s=READY pgs=34 cs=0 l=1 rev1=1 crypto rx=0x7f6d80005fd0 tx=0x7f6d8000b690 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:20.725 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.720+0000 7f6d857fa640 1 -- 172.21.15.78:0/1217016011 <== mon.1 v2:172.21.15.78:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6d8001e070 con 0x7f6d8812bdb0 2024-09-21T09:22:20.725 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.720+0000 7f6d857fa640 1 -- 172.21.15.78:0/1217016011 <== mon.1 v2:172.21.15.78:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f6d8000be40 con 0x7f6d8812bdb0 2024-09-21T09:22:20.725 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.720+0000 7f6d8f1c5640 1 -- 172.21.15.78:0/1217016011 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f6d88133140 con 0x7f6d8812bdb0 2024-09-21T09:22:20.725 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.720+0000 7f6d857fa640 1 -- 172.21.15.78:0/1217016011 <== mon.1 v2:172.21.15.78:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6d80004290 con 0x7f6d8812bdb0 2024-09-21T09:22:20.725 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.720+0000 7f6d8f1c5640 1 -- 172.21.15.78:0/1217016011 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7f6d8812c830 con 0x7f6d8812bdb0 2024-09-21T09:22:20.725 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.721+0000 7f6d8f1c5640 1 -- 172.21.15.78:0/1217016011 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f6d880959e0 con 0x7f6d8812bdb0 2024-09-21T09:22:20.725 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.721+0000 7f6d857fa640 1 -- 172.21.15.78:0/1217016011 <== mon.1 v2:172.21.15.78:3301/0 4 ==== mgrmap(e 9) ==== 63442+0+0 (secure 0 0 0) 0x7f6d80021070 con 0x7f6d8812bdb0 2024-09-21T09:22:20.725 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.722+0000 7f6d857fa640 1 -- 172.21.15.78:0/1217016011 <== mon.1 v2:172.21.15.78:3301/0 5 ==== osd_map(20..20 src has 1..20) ==== 3027+0+0 (secure 0 0 0) 0x7f6d8006cd30 con 0x7f6d8812bdb0 2024-09-21T09:22:20.727 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.725+0000 7f6d857fa640 1 -- 172.21.15.78:0/1217016011 <== mon.1 v2:172.21.15.78:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f6d80071070 con 0x7f6d8812bdb0 2024-09-21T09:22:20.884 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.882+0000 7f6d8f1c5640 1 -- 172.21.15.78:0/1217016011 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_command({"prefix": "osd down", "ids": ["0"]} v 0) -- 0x7f6d880699c0 con 0x7f6d8812bdb0 2024-09-21T09:22:20.885 INFO:tasks.ceph.mon.a.smithi078.stderr:2024-09-21T09:22:20.883+0000 7f632064c640 -1 mon.a@0(leader).osd e20 definitely_dead 0 2024-09-21T09:22:20.925 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.922+0000 7f6d857fa640 1 -- 172.21.15.78:0/1217016011 <== mon.1 v2:172.21.15.78:3301/0 7 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f6d8006b990 con 0x7f6d8812bdb0 2024-09-21T09:22:20.925 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.923+0000 7f6d857fa640 1 -- 172.21.15.78:0/1217016011 <== mon.1 v2:172.21.15.78:3301/0 8 ==== mon_command_ack([{"prefix": "osd down", "ids": ["0"]}]=0 marked down osd.0. v21) ==== 89+0+0 (secure 0 0 0) 0x7f6d80067a90 con 0x7f6d8812bdb0 2024-09-21T09:22:20.925 INFO:teuthology.orchestra.run.smithi078.stderr:marked down osd.0. 2024-09-21T09:22:20.927 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.924+0000 7f6d8f1c5640 1 -- 172.21.15.78:0/1217016011 >> v1:172.21.15.78:6816/2186157269 conn(0x7f6d68051420 legacy=0x7f6d680538e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:20.927 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.924+0000 7f6d8f1c5640 1 -- 172.21.15.78:0/1217016011 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f6d8812bdb0 msgr2=0x7f6d88134f10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:20.927 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.924+0000 7f6d8f1c5640 1 --2- 172.21.15.78:0/1217016011 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f6d8812bdb0 0x7f6d88134f10 secure :-1 s=READY pgs=34 cs=0 l=1 rev1=1 crypto rx=0x7f6d80005fd0 tx=0x7f6d8000b690 comp rx=0 tx=0).stop 2024-09-21T09:22:20.927 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.925+0000 7f6d8f1c5640 1 -- 172.21.15.78:0/1217016011 shutdown_connections 2024-09-21T09:22:20.927 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.925+0000 7f6d8f1c5640 1 --2- 172.21.15.78:0/1217016011 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f6d8812bdb0 0x7f6d88134f10 unknown :-1 s=CLOSED pgs=34 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:20.927 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.925+0000 7f6d8f1c5640 1 --2- 172.21.15.78:0/1217016011 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f6d8812b490 0x7f6d881349d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:20.928 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.925+0000 7f6d8f1c5640 1 --2- 172.21.15.78:0/1217016011 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f6d88095570 0x7f6d88134490 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:20.928 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.925+0000 7f6d8f1c5640 1 -- 172.21.15.78:0/1217016011 >> 172.21.15.78:0/1217016011 conn(0x7f6d8809afd0 msgr2=0x7f6d88091cd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:20.928 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.925+0000 7f6d8f1c5640 1 -- 172.21.15.78:0/1217016011 shutdown_connections 2024-09-21T09:22:20.928 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:20.925+0000 7f6d8f1c5640 1 -- 172.21.15.78:0/1217016011 wait complete. 2024-09-21T09:22:20.937 INFO:tasks.ceph.osd.0:Restarting daemon 2024-09-21T09:22:20.938 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 0 2024-09-21T09:22:20.939 INFO:tasks.ceph.osd.0:Started 2024-09-21T09:22:20.940 DEBUG:tasks.ceph.osd.1:waiting for process to exit 2024-09-21T09:22:20.940 INFO:teuthology.orchestra.run:waiting for 300 2024-09-21T09:22:20.981 INFO:tasks.ceph.osd.1:Stopped 2024-09-21T09:22:20.981 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd down 1 2024-09-21T09:22:21.082 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:21.079+0000 7faf16978640 1 -- 172.21.15.78:0/1213162209 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7faf10128970 msgr2=0x7faf10133b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:21.082 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:21.079+0000 7faf16978640 1 --2- 172.21.15.78:0/1213162209 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7faf10128970 0x7faf10133b60 secure :-1 s=READY pgs=27 cs=0 l=1 rev1=1 crypto rx=0x7faf08005fd0 tx=0x7faf0802a820 comp rx=0 tx=0).stop 2024-09-21T09:22:21.082 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:21.079+0000 7faf16978640 1 -- 172.21.15.78:0/1213162209 shutdown_connections 2024-09-21T09:22:21.082 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:21.079+0000 7faf16978640 1 --2- 172.21.15.78:0/1213162209 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7faf10128970 0x7faf10133b60 unknown :-1 s=CLOSED pgs=27 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:21.082 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:21.079+0000 7faf16978640 1 --2- 172.21.15.78:0/1213162209 >> v2:172.21.15.78:3300/0 conn(0x7faf1012bad0 0x7faf1012beb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:21.082 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:21.079+0000 7faf16978640 1 -- 172.21.15.78:0/1213162209 >> 172.21.15.78:0/1213162209 conn(0x7faf1009afd0 msgr2=0x7faf1009b3e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:21.082 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:21.079+0000 7faf16978640 1 -- 172.21.15.78:0/1213162209 shutdown_connections 2024-09-21T09:22:21.083 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:21.079+0000 7faf16978640 1 -- 172.21.15.78:0/1213162209 wait complete. 2024-09-21T09:22:21.085 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:21.080+0000 7faf16978640 1 Processor -- start 2024-09-21T09:22:21.085 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:21.080+0000 7faf16978640 1 -- start start 2024-09-21T09:22:21.085 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:21.081+0000 7faf16978640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7faf10128970 0x7faf10072ee0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:21.085 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:21.081+0000 7faf16978640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7faf1012bad0 0x7faf10073420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:21.085 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:21.081+0000 7faf16978640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7faf1012c3f0 0x7faf10073960 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:21.085 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:21.081+0000 7faf16978640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7faf10134680 con 0x7faf1012c3f0 2024-09-21T09:22:21.085 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:21.081+0000 7faf16978640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7faf10134500 con 0x7faf10128970 2024-09-21T09:22:21.085 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:21.081+0000 7faf16978640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7faf10134800 con 0x7faf1012bad0 2024-09-21T09:22:21.085 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:21.081+0000 7faf14f15640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7faf1012c3f0 0x7faf10073960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:21.086 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:21.081+0000 7faf0ffff640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7faf10128970 0x7faf10072ee0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:21.086 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:21.081+0000 7faf14f15640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7faf1012c3f0 0x7faf10073960 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3300/0 says I am v2:172.21.15.78:51568/0 (socket says 172.21.15.78:51568) 2024-09-21T09:22:21.086 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:21.081+0000 7faf0ffff640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7faf10128970 0x7faf10072ee0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3301/0 says I am v2:172.21.15.78:32774/0 (socket says 172.21.15.78:32774) 2024-09-21T09:22:21.086 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:21.081+0000 7faf14f15640 1 -- 172.21.15.78:0/3782438218 learned_addr learned my addr 172.21.15.78:0/3782438218 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:21.086 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:21.081+0000 7faf0f7fe640 1 --2- 172.21.15.78:0/3782438218 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7faf1012bad0 0x7faf10073420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:21.086 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:21.081+0000 7faf14f15640 1 -- 172.21.15.78:0/3782438218 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7faf1012bad0 msgr2=0x7faf10073420 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:21.086 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:21.081+0000 7faf14f15640 1 --2- 172.21.15.78:0/3782438218 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7faf1012bad0 0x7faf10073420 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:21.086 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:21.081+0000 7faf14f15640 1 -- 172.21.15.78:0/3782438218 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7faf10128970 msgr2=0x7faf10072ee0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:21.086 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:21.081+0000 7faf14f15640 1 --2- 172.21.15.78:0/3782438218 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7faf10128970 0x7faf10072ee0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:21.086 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:21.081+0000 7faf14f15640 1 -- 172.21.15.78:0/3782438218 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7faf10070f90 con 0x7faf1012c3f0 2024-09-21T09:22:21.086 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:21.081+0000 7faf0ffff640 1 --2- 172.21.15.78:0/3782438218 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7faf10128970 0x7faf10072ee0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:22:21.086 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:21.081+0000 7faf14f15640 1 --2- 172.21.15.78:0/3782438218 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7faf1012c3f0 0x7faf10073960 secure :-1 s=READY pgs=73 cs=0 l=1 rev1=1 crypto rx=0x7faf08035040 tx=0x7faf08032c60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:21.086 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:21.081+0000 7faf0cff9640 1 -- 172.21.15.78:0/3782438218 <== mon.0 v2:172.21.15.78:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7faf08037040 con 0x7faf1012c3f0 2024-09-21T09:22:21.087 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:21.081+0000 7faf0cff9640 1 -- 172.21.15.78:0/3782438218 <== mon.0 v2:172.21.15.78:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7faf08042070 con 0x7faf1012c3f0 2024-09-21T09:22:21.087 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:21.081+0000 7faf0cff9640 1 -- 172.21.15.78:0/3782438218 <== mon.0 v2:172.21.15.78:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7faf08003e60 con 0x7faf1012c3f0 2024-09-21T09:22:21.087 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:21.081+0000 7faf16978640 1 -- 172.21.15.78:0/3782438218 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7faf10072c20 con 0x7faf1012c3f0 2024-09-21T09:22:21.087 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:21.081+0000 7faf16978640 1 -- 172.21.15.78:0/3782438218 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7faf10133be0 con 0x7faf1012c3f0 2024-09-21T09:22:21.087 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:21.082+0000 7faf16978640 1 -- 172.21.15.78:0/3782438218 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7faf1012beb0 con 0x7faf1012c3f0 2024-09-21T09:22:21.087 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:21.083+0000 7faf0cff9640 1 -- 172.21.15.78:0/3782438218 <== mon.0 v2:172.21.15.78:3300/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7faf08004260 con 0x7faf1012c3f0 2024-09-21T09:22:21.087 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:21.083+0000 7faf0cff9640 1 -- 172.21.15.78:0/3782438218 <== mon.0 v2:172.21.15.78:3300/0 5 ==== osd_map(21..21 src has 1..21) ==== 3027+0+0 (secure 0 0 0) 0x7faf08089b80 con 0x7faf1012c3f0 2024-09-21T09:22:21.089 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:21.086+0000 7faf0cff9640 1 -- 172.21.15.78:0/3782438218 <== mon.0 v2:172.21.15.78:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7faf08088c30 con 0x7faf1012c3f0 2024-09-21T09:22:21.248 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:21.246+0000 7faf16978640 1 -- 172.21.15.78:0/3782438218 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "osd down", "ids": ["1"]} v 0) -- 0x7faf10071e40 con 0x7faf1012c3f0 2024-09-21T09:22:21.249 INFO:tasks.ceph.mon.a.smithi078.stderr:2024-09-21T09:22:21.246+0000 7f632064c640 -1 mon.a@0(leader).osd e21 definitely_dead 0 2024-09-21T09:22:21.920 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:21.917+0000 7faf0cff9640 1 -- 172.21.15.78:0/3782438218 <== mon.0 v2:172.21.15.78:3300/0 7 ==== mon_command_ack([{"prefix": "osd down", "ids": ["1"]}]=0 marked down osd.1. v22) ==== 89+0+0 (secure 0 0 0) 0x7faf08032450 con 0x7faf1012c3f0 2024-09-21T09:22:21.920 INFO:teuthology.orchestra.run.smithi078.stderr:marked down osd.1. 2024-09-21T09:22:21.923 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:21.920+0000 7faf16978640 1 -- 172.21.15.78:0/3782438218 >> v1:172.21.15.78:6816/2186157269 conn(0x7faefc0514b0 legacy=0x7faefc053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:21.923 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:21.920+0000 7faf16978640 1 -- 172.21.15.78:0/3782438218 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7faf1012c3f0 msgr2=0x7faf10073960 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:21.923 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:21.920+0000 7faf16978640 1 --2- 172.21.15.78:0/3782438218 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7faf1012c3f0 0x7faf10073960 secure :-1 s=READY pgs=73 cs=0 l=1 rev1=1 crypto rx=0x7faf08035040 tx=0x7faf08032c60 comp rx=0 tx=0).stop 2024-09-21T09:22:21.923 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:21.920+0000 7faf16978640 1 -- 172.21.15.78:0/3782438218 shutdown_connections 2024-09-21T09:22:21.923 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:21.920+0000 7faf16978640 1 --2- 172.21.15.78:0/3782438218 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7faf1012c3f0 0x7faf10073960 unknown :-1 s=CLOSED pgs=73 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:21.923 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:21.920+0000 7faf16978640 1 --2- 172.21.15.78:0/3782438218 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7faf1012bad0 0x7faf10073420 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:21.923 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:21.920+0000 7faf16978640 1 --2- 172.21.15.78:0/3782438218 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7faf10128970 0x7faf10072ee0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:21.923 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:21.920+0000 7faf16978640 1 -- 172.21.15.78:0/3782438218 >> 172.21.15.78:0/3782438218 conn(0x7faf1009afd0 msgr2=0x7faf10098850 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:21.923 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:21.920+0000 7faf16978640 1 -- 172.21.15.78:0/3782438218 shutdown_connections 2024-09-21T09:22:21.923 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:21.920+0000 7faf16978640 1 -- 172.21.15.78:0/3782438218 wait complete. 2024-09-21T09:22:21.935 INFO:tasks.ceph.osd.1:Restarting daemon 2024-09-21T09:22:21.935 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 1 2024-09-21T09:22:21.937 INFO:tasks.ceph.osd.1:Started 2024-09-21T09:22:21.938 DEBUG:tasks.ceph.osd.2:waiting for process to exit 2024-09-21T09:22:21.938 INFO:teuthology.orchestra.run:waiting for 300 2024-09-21T09:22:21.982 INFO:tasks.ceph.osd.2:Stopped 2024-09-21T09:22:21.983 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd down 2 2024-09-21T09:22:22.084 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:22.082+0000 7f281aa52640 1 -- 172.21.15.78:0/1722061689 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f2814098d40 msgr2=0x7f2814099120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:22.085 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:22.082+0000 7f281aa52640 1 --2- 172.21.15.78:0/1722061689 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f2814098d40 0x7f2814099120 secure :-1 s=READY pgs=75 cs=0 l=1 rev1=1 crypto rx=0x7f2804009830 tx=0x7f280402a360 comp rx=0 tx=0).stop 2024-09-21T09:22:22.085 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:22.082+0000 7f281aa52640 1 -- 172.21.15.78:0/1722061689 shutdown_connections 2024-09-21T09:22:22.085 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:22.082+0000 7f281aa52640 1 --2- 172.21.15.78:0/1722061689 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f281412eff0 0x7f281412f480 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:22.085 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:22.082+0000 7f281aa52640 1 --2- 172.21.15.78:0/1722061689 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f2814099660 0x7f281412eab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:22.085 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:22.082+0000 7f281aa52640 1 --2- 172.21.15.78:0/1722061689 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f2814098d40 0x7f2814099120 unknown :-1 s=CLOSED pgs=75 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:22.085 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:22.082+0000 7f281aa52640 1 -- 172.21.15.78:0/1722061689 >> 172.21.15.78:0/1722061689 conn(0x7f28140a1790 msgr2=0x7f28140a1ba0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:22.085 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:22.082+0000 7f281aa52640 1 -- 172.21.15.78:0/1722061689 shutdown_connections 2024-09-21T09:22:22.085 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:22.082+0000 7f281aa52640 1 -- 172.21.15.78:0/1722061689 wait complete. 2024-09-21T09:22:22.087 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:22.083+0000 7f281aa52640 1 Processor -- start 2024-09-21T09:22:22.087 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:22.083+0000 7f281aa52640 1 -- start start 2024-09-21T09:22:22.088 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:22.083+0000 7f281aa52640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f2814098d40 0x7f28141c4cb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:22.088 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:22.083+0000 7f281aa52640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f2814099660 0x7f28141c51f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:22.088 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:22.083+0000 7f281aa52640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f281412eff0 0x7f2814132910 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:22.088 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:22.083+0000 7f281aa52640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7f28141365b0 con 0x7f2814098d40 2024-09-21T09:22:22.088 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:22.083+0000 7f281aa52640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7f2814136430 con 0x7f281412eff0 2024-09-21T09:22:22.088 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:22.083+0000 7f281aa52640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7f2814136730 con 0x7f2814099660 2024-09-21T09:22:22.088 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:22.083+0000 7f2813fff640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f2814098d40 0x7f28141c4cb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:22.088 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:22.083+0000 7f280bfff640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f2814099660 0x7f28141c51f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:22.088 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:22.083+0000 7f2813fff640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f2814098d40 0x7f28141c4cb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3300/0 says I am v2:172.21.15.78:51586/0 (socket says 172.21.15.78:51586) 2024-09-21T09:22:22.088 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:22.083+0000 7f280bfff640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f2814099660 0x7f28141c51f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3302/0 says I am v2:172.21.15.78:54136/0 (socket says 172.21.15.78:54136) 2024-09-21T09:22:22.088 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:22.083+0000 7f2818fef640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f281412eff0 0x7f2814132910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:22.088 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:22.083+0000 7f2813fff640 1 -- 172.21.15.78:0/1815391341 learned_addr learned my addr 172.21.15.78:0/1815391341 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:22.088 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:22.084+0000 7f2813fff640 1 -- 172.21.15.78:0/1815391341 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f2814099660 msgr2=0x7f28141c51f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:22.089 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:22.084+0000 7f2813fff640 1 --2- 172.21.15.78:0/1815391341 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f2814099660 0x7f28141c51f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:22.089 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:22.084+0000 7f2813fff640 1 -- 172.21.15.78:0/1815391341 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f281412eff0 msgr2=0x7f2814132910 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:22.089 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:22.084+0000 7f2813fff640 1 --2- 172.21.15.78:0/1815391341 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f281412eff0 0x7f2814132910 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:22.089 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:22.084+0000 7f2813fff640 1 -- 172.21.15.78:0/1815391341 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f2814134930 con 0x7f2814098d40 2024-09-21T09:22:22.089 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:22.084+0000 7f2818fef640 1 --2- 172.21.15.78:0/1815391341 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f281412eff0 0x7f2814132910 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:22:22.089 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:22.084+0000 7f2813fff640 1 --2- 172.21.15.78:0/1815391341 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f2814098d40 0x7f28141c4cb0 secure :-1 s=READY pgs=76 cs=0 l=1 rev1=1 crypto rx=0x7f2804009950 tx=0x7f2804033b70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:22.089 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:22.084+0000 7f28117fa640 1 -- 172.21.15.78:0/1815391341 <== mon.0 v2:172.21.15.78:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f280403f020 con 0x7f2814098d40 2024-09-21T09:22:22.089 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:22.084+0000 7f28117fa640 1 -- 172.21.15.78:0/1815391341 <== mon.0 v2:172.21.15.78:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f2804033d60 con 0x7f2814098d40 2024-09-21T09:22:22.089 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:22.084+0000 7f281aa52640 1 -- 172.21.15.78:0/1815391341 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f2814129460 con 0x7f2814098d40 2024-09-21T09:22:22.089 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:22.084+0000 7f28117fa640 1 -- 172.21.15.78:0/1815391341 <== mon.0 v2:172.21.15.78:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2804037040 con 0x7f2814098d40 2024-09-21T09:22:22.089 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:22.084+0000 7f281aa52640 1 -- 172.21.15.78:0/1815391341 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f2814132e50 con 0x7f2814098d40 2024-09-21T09:22:22.089 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:22.085+0000 7f281aa52640 1 -- 172.21.15.78:0/1815391341 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f281412eab0 con 0x7f2814098d40 2024-09-21T09:22:22.089 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:22.085+0000 7f28117fa640 1 -- 172.21.15.78:0/1815391341 <== mon.0 v2:172.21.15.78:3300/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f2804042020 con 0x7f2814098d40 2024-09-21T09:22:22.090 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:22.086+0000 7f28117fa640 1 -- 172.21.15.78:0/1815391341 <== mon.0 v2:172.21.15.78:3300/0 5 ==== osd_map(22..22 src has 1..22) ==== 3027+0+0 (secure 0 0 0) 0x7f2804088da0 con 0x7f2814098d40 2024-09-21T09:22:22.091 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:22.089+0000 7f28117fa640 1 -- 172.21.15.78:0/1815391341 <== mon.0 v2:172.21.15.78:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f2804049030 con 0x7f2814098d40 2024-09-21T09:22:22.251 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:22.249+0000 7f281aa52640 1 -- 172.21.15.78:0/1815391341 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "osd down", "ids": ["2"]} v 0) -- 0x7f28140698c0 con 0x7f2814098d40 2024-09-21T09:22:22.251 INFO:tasks.ceph.mon.a.smithi078.stderr:2024-09-21T09:22:22.249+0000 7f632064c640 -1 mon.a@0(leader).osd e22 definitely_dead 0 2024-09-21T09:22:22.363 INFO:tasks.ceph.osd.0.smithi078.stderr:2024-09-21T09:22:22.361+0000 7fb036b34640 -1 Falling back to public interface 2024-09-21T09:22:22.924 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:22.921+0000 7f28117fa640 1 -- 172.21.15.78:0/1815391341 <== mon.0 v2:172.21.15.78:3300/0 7 ==== mon_command_ack([{"prefix": "osd down", "ids": ["2"]}]=0 marked down osd.2. v23) ==== 89+0+0 (secure 0 0 0) 0x7f2804087e50 con 0x7f2814098d40 2024-09-21T09:22:22.924 INFO:teuthology.orchestra.run.smithi078.stderr:marked down osd.2. 2024-09-21T09:22:22.927 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:22.924+0000 7f281aa52640 1 -- 172.21.15.78:0/1815391341 >> v1:172.21.15.78:6816/2186157269 conn(0x7f27f40514b0 legacy=0x7f27f4053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:22.927 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:22.924+0000 7f281aa52640 1 -- 172.21.15.78:0/1815391341 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f2814098d40 msgr2=0x7f28141c4cb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:22.927 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:22.924+0000 7f281aa52640 1 --2- 172.21.15.78:0/1815391341 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f2814098d40 0x7f28141c4cb0 secure :-1 s=READY pgs=76 cs=0 l=1 rev1=1 crypto rx=0x7f2804009950 tx=0x7f2804033b70 comp rx=0 tx=0).stop 2024-09-21T09:22:22.927 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:22.924+0000 7f281aa52640 1 -- 172.21.15.78:0/1815391341 shutdown_connections 2024-09-21T09:22:22.927 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:22.924+0000 7f281aa52640 1 --2- 172.21.15.78:0/1815391341 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f281412eff0 0x7f2814132910 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:22.927 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:22.924+0000 7f281aa52640 1 --2- 172.21.15.78:0/1815391341 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f2814099660 0x7f28141c51f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:22.927 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:22.924+0000 7f281aa52640 1 --2- 172.21.15.78:0/1815391341 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f2814098d40 0x7f28141c4cb0 secure :-1 s=CLOSED pgs=76 cs=0 l=1 rev1=1 crypto rx=0x7f2804009950 tx=0x7f2804033b70 comp rx=0 tx=0).stop 2024-09-21T09:22:22.927 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:22.924+0000 7f281aa52640 1 -- 172.21.15.78:0/1815391341 >> 172.21.15.78:0/1815391341 conn(0x7f28140a1790 msgr2=0x7f2814128440 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:22.928 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:22.925+0000 7f281aa52640 1 -- 172.21.15.78:0/1815391341 shutdown_connections 2024-09-21T09:22:22.928 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:22.925+0000 7f281aa52640 1 -- 172.21.15.78:0/1815391341 wait complete. 2024-09-21T09:22:22.939 INFO:tasks.ceph.osd.2:Restarting daemon 2024-09-21T09:22:22.939 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 2 2024-09-21T09:22:22.942 INFO:tasks.ceph.osd.2:Started 2024-09-21T09:22:22.942 DEBUG:tasks.ceph.osd.3:waiting for process to exit 2024-09-21T09:22:22.942 INFO:teuthology.orchestra.run:waiting for 300 2024-09-21T09:22:22.973 INFO:tasks.ceph.osd.3:Stopped 2024-09-21T09:22:22.973 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd down 3 2024-09-21T09:22:23.114 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:23.111+0000 7faf25a45640 1 -- 172.21.15.78:0/503914577 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7faf2012f1c0 msgr2=0x7faf2012f600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:23.114 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:23.111+0000 7faf25a45640 1 --2- 172.21.15.78:0/503914577 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7faf2012f1c0 0x7faf2012f600 secure :-1 s=READY pgs=79 cs=0 l=1 rev1=1 crypto rx=0x7faf18005fd0 tx=0x7faf1802a780 comp rx=0 tx=0).stop 2024-09-21T09:22:23.114 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:23.111+0000 7faf25a45640 1 -- 172.21.15.78:0/503914577 shutdown_connections 2024-09-21T09:22:23.114 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:23.111+0000 7faf25a45640 1 --2- 172.21.15.78:0/503914577 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7faf2012f1c0 0x7faf2012f600 unknown :-1 s=CLOSED pgs=79 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:23.114 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:23.111+0000 7faf25a45640 1 --2- 172.21.15.78:0/503914577 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7faf20098d40 0x7faf20099120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:23.114 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:23.111+0000 7faf25a45640 1 -- 172.21.15.78:0/503914577 >> 172.21.15.78:0/503914577 conn(0x7faf200a1790 msgr2=0x7faf200a1ba0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:23.114 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:23.111+0000 7faf25a45640 1 -- 172.21.15.78:0/503914577 shutdown_connections 2024-09-21T09:22:23.114 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:23.111+0000 7faf25a45640 1 -- 172.21.15.78:0/503914577 wait complete. 2024-09-21T09:22:23.117 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:23.112+0000 7faf25a45640 1 Processor -- start 2024-09-21T09:22:23.117 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:23.112+0000 7faf25a45640 1 -- start start 2024-09-21T09:22:23.117 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:23.112+0000 7faf25a45640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7faf20098d40 0x7faf201c4cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:23.117 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:23.112+0000 7faf25a45640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7faf20099660 0x7faf201c5200 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:23.117 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:23.112+0000 7faf25a45640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7faf2012f1c0 0x7faf201c5740 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:23.117 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:23.112+0000 7faf25a45640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7faf20137870 con 0x7faf20099660 2024-09-21T09:22:23.117 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:23.112+0000 7faf25a45640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7faf201376f0 con 0x7faf2012f1c0 2024-09-21T09:22:23.117 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:23.112+0000 7faf25a45640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7faf201379f0 con 0x7faf20098d40 2024-09-21T09:22:23.117 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:23.112+0000 7faf1effd640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7faf20098d40 0x7faf201c4cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:23.117 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:23.112+0000 7faf17fff640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7faf20099660 0x7faf201c5200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:23.117 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:23.112+0000 7faf1f7fe640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7faf2012f1c0 0x7faf201c5740 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:23.118 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:23.112+0000 7faf1effd640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7faf20098d40 0x7faf201c4cc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3302/0 says I am v2:172.21.15.78:45326/0 (socket says 172.21.15.78:45326) 2024-09-21T09:22:23.118 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:23.113+0000 7faf1effd640 1 -- 172.21.15.78:0/3247877863 learned_addr learned my addr 172.21.15.78:0/3247877863 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:23.118 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:23.113+0000 7faf1effd640 1 -- 172.21.15.78:0/3247877863 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7faf2012f1c0 msgr2=0x7faf201c5740 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:23.118 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:23.113+0000 7faf1effd640 1 --2- 172.21.15.78:0/3247877863 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7faf2012f1c0 0x7faf201c5740 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:23.118 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:23.113+0000 7faf1effd640 1 -- 172.21.15.78:0/3247877863 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7faf20099660 msgr2=0x7faf201c5200 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:23.118 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:23.113+0000 7faf1effd640 1 --2- 172.21.15.78:0/3247877863 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7faf20099660 0x7faf201c5200 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:23.118 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:23.113+0000 7faf1effd640 1 -- 172.21.15.78:0/3247877863 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7faf20073820 con 0x7faf20098d40 2024-09-21T09:22:23.118 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:23.113+0000 7faf17fff640 1 --2- 172.21.15.78:0/3247877863 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7faf20099660 0x7faf201c5200 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:22:23.118 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:23.113+0000 7faf1effd640 1 --2- 172.21.15.78:0/3247877863 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7faf20098d40 0x7faf201c4cc0 secure :-1 s=READY pgs=28 cs=0 l=1 rev1=1 crypto rx=0x7faf1000e7c0 tx=0x7faf1000ec80 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:23.118 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:23.113+0000 7faf1cff9640 1 -- 172.21.15.78:0/3247877863 <== mon.2 v2:172.21.15.78:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7faf1000ccc0 con 0x7faf20098d40 2024-09-21T09:22:23.118 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:23.113+0000 7faf1cff9640 1 -- 172.21.15.78:0/3247877863 <== mon.2 v2:172.21.15.78:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7faf10010430 con 0x7faf20098d40 2024-09-21T09:22:23.118 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:23.113+0000 7faf25a45640 1 -- 172.21.15.78:0/3247877863 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7faf2012ff10 con 0x7faf20098d40 2024-09-21T09:22:23.118 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:23.113+0000 7faf1cff9640 1 -- 172.21.15.78:0/3247877863 <== mon.2 v2:172.21.15.78:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7faf10010860 con 0x7faf20098d40 2024-09-21T09:22:23.118 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:23.113+0000 7faf25a45640 1 -- 172.21.15.78:0/3247877863 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7faf20071770 con 0x7faf20098d40 2024-09-21T09:22:23.119 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:23.114+0000 7faf25a45640 1 -- 172.21.15.78:0/3247877863 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7faf2012d030 con 0x7faf20098d40 2024-09-21T09:22:23.119 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:23.114+0000 7faf1cff9640 1 -- 172.21.15.78:0/3247877863 <== mon.2 v2:172.21.15.78:3302/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7faf10010a00 con 0x7faf20098d40 2024-09-21T09:22:23.119 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:23.115+0000 7faf1cff9640 1 -- 172.21.15.78:0/3247877863 <== mon.2 v2:172.21.15.78:3302/0 5 ==== osd_map(23..23 src has 1..23) ==== 3027+0+0 (secure 0 0 0) 0x7faf100699b0 con 0x7faf20098d40 2024-09-21T09:22:23.121 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:23.119+0000 7faf1cff9640 1 -- 172.21.15.78:0/3247877863 <== mon.2 v2:172.21.15.78:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7faf2012d030 con 0x7faf20098d40 2024-09-21T09:22:23.123 INFO:tasks.ceph.osd.1.smithi078.stderr:2024-09-21T09:22:23.121+0000 7f640d3dd640 -1 Falling back to public interface 2024-09-21T09:22:23.280 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:23.278+0000 7faf25a45640 1 -- 172.21.15.78:0/3247877863 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_command({"prefix": "osd down", "ids": ["3"]} v 0) -- 0x7faf2012d450 con 0x7faf20098d40 2024-09-21T09:22:23.280 INFO:tasks.ceph.mon.a.smithi078.stderr:2024-09-21T09:22:23.278+0000 7f632064c640 -1 mon.a@0(leader).osd e23 definitely_dead 0 2024-09-21T09:22:23.867 INFO:tasks.ceph.osd.2.smithi078.stderr:2024-09-21T09:22:23.864+0000 7f36f0f3e640 -1 Falling back to public interface 2024-09-21T09:22:23.929 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:23.927+0000 7faf1cff9640 1 -- 172.21.15.78:0/3247877863 <== mon.2 v2:172.21.15.78:3302/0 7 ==== mon_command_ack([{"prefix": "osd down", "ids": ["3"]}]=0 marked down osd.3. v24) ==== 89+0+0 (secure 0 0 0) 0x7faf2012d450 con 0x7faf20098d40 2024-09-21T09:22:23.929 INFO:teuthology.orchestra.run.smithi078.stderr:marked down osd.3. 2024-09-21T09:22:23.932 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:23.929+0000 7faf25a45640 1 -- 172.21.15.78:0/3247877863 >> v1:172.21.15.78:6816/2186157269 conn(0x7faf000514b0 legacy=0x7faf00053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:23.932 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:23.929+0000 7faf25a45640 1 -- 172.21.15.78:0/3247877863 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7faf20098d40 msgr2=0x7faf201c4cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:23.932 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:23.929+0000 7faf25a45640 1 --2- 172.21.15.78:0/3247877863 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7faf20098d40 0x7faf201c4cc0 secure :-1 s=READY pgs=28 cs=0 l=1 rev1=1 crypto rx=0x7faf1000e7c0 tx=0x7faf1000ec80 comp rx=0 tx=0).stop 2024-09-21T09:22:23.932 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:23.929+0000 7faf25a45640 1 -- 172.21.15.78:0/3247877863 shutdown_connections 2024-09-21T09:22:23.932 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:23.929+0000 7faf25a45640 1 --2- 172.21.15.78:0/3247877863 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7faf2012f1c0 0x7faf201c5740 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:23.932 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:23.929+0000 7faf25a45640 1 --2- 172.21.15.78:0/3247877863 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7faf20099660 0x7faf201c5200 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:23.932 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:23.929+0000 7faf25a45640 1 --2- 172.21.15.78:0/3247877863 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7faf20098d40 0x7faf201c4cc0 unknown :-1 s=CLOSED pgs=28 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:23.932 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:23.929+0000 7faf25a45640 1 -- 172.21.15.78:0/3247877863 >> 172.21.15.78:0/3247877863 conn(0x7faf200a1790 msgr2=0x7faf2009ce40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:23.932 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:23.929+0000 7faf25a45640 1 -- 172.21.15.78:0/3247877863 shutdown_connections 2024-09-21T09:22:23.933 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:23.929+0000 7faf25a45640 1 -- 172.21.15.78:0/3247877863 wait complete. 2024-09-21T09:22:23.944 INFO:tasks.ceph.osd.3:Restarting daemon 2024-09-21T09:22:23.944 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 3 2024-09-21T09:22:23.946 INFO:tasks.ceph.osd.3:Started 2024-09-21T09:22:23.946 INFO:tasks.ceph.ceph_manager.ceph:waiting for all up 2024-09-21T09:22:23.947 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-09-21T09:22:24.049 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:24.046+0000 7efd355a1640 1 -- 172.21.15.78:0/1124114826 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7efd30129540 msgr2=0x7efd30129920 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:24.049 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:24.046+0000 7efd355a1640 1 --2- 172.21.15.78:0/1124114826 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7efd30129540 0x7efd30129920 secure :-1 s=READY pgs=29 cs=0 l=1 rev1=1 crypto rx=0x7efd20009830 tx=0x7efd2002a3f0 comp rx=0 tx=0).stop 2024-09-21T09:22:24.049 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:24.046+0000 7efd355a1640 1 -- 172.21.15.78:0/1124114826 shutdown_connections 2024-09-21T09:22:24.049 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:24.046+0000 7efd355a1640 1 --2- 172.21.15.78:0/1124114826 >> v2:172.21.15.78:3300/0 conn(0x7efd30129e60 0x7efd3012c850 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:24.049 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:24.046+0000 7efd355a1640 1 --2- 172.21.15.78:0/1124114826 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7efd30129540 0x7efd30129920 unknown :-1 s=CLOSED pgs=29 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:24.049 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:24.046+0000 7efd355a1640 1 -- 172.21.15.78:0/1124114826 >> 172.21.15.78:0/1124114826 conn(0x7efd3009d480 msgr2=0x7efd3009d890 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:24.050 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:24.046+0000 7efd355a1640 1 -- 172.21.15.78:0/1124114826 shutdown_connections 2024-09-21T09:22:24.050 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:24.046+0000 7efd355a1640 1 -- 172.21.15.78:0/1124114826 wait complete. 2024-09-21T09:22:24.052 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:24.047+0000 7efd355a1640 1 Processor -- start 2024-09-21T09:22:24.052 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:24.047+0000 7efd355a1640 1 -- start start 2024-09-21T09:22:24.052 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:24.048+0000 7efd355a1640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7efd30129540 0x7efd301c4ef0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:24.052 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:24.048+0000 7efd355a1640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7efd30129e60 0x7efd301c5430 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:24.052 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:24.048+0000 7efd355a1640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7efd3012cd90 0x7efd3006f3e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:24.052 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:24.048+0000 7efd355a1640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7efd30135810 con 0x7efd3012cd90 2024-09-21T09:22:24.052 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:24.048+0000 7efd355a1640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7efd30135690 con 0x7efd30129e60 2024-09-21T09:22:24.052 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:24.048+0000 7efd355a1640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7efd30135990 con 0x7efd30129540 2024-09-21T09:22:24.052 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:24.048+0000 7efd2effd640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7efd30129540 0x7efd301c4ef0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:24.052 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:24.048+0000 7efd2f7fe640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7efd3012cd90 0x7efd3006f3e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:24.052 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:24.048+0000 7efd2effd640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7efd30129540 0x7efd301c4ef0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3302/0 says I am v2:172.21.15.78:45358/0 (socket says 172.21.15.78:45358) 2024-09-21T09:22:24.052 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:24.048+0000 7efd2e7fc640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7efd30129e60 0x7efd301c5430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:24.053 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:24.048+0000 7efd2effd640 1 -- 172.21.15.78:0/1086448712 learned_addr learned my addr 172.21.15.78:0/1086448712 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:24.053 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:24.048+0000 7efd2f7fe640 1 -- 172.21.15.78:0/1086448712 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7efd30129540 msgr2=0x7efd301c4ef0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:24.053 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:24.048+0000 7efd2f7fe640 1 --2- 172.21.15.78:0/1086448712 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7efd30129540 0x7efd301c4ef0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:24.053 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:24.048+0000 7efd2f7fe640 1 -- 172.21.15.78:0/1086448712 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7efd30129e60 msgr2=0x7efd301c5430 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:24.053 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:24.048+0000 7efd2f7fe640 1 --2- 172.21.15.78:0/1086448712 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7efd30129e60 0x7efd301c5430 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:24.053 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:24.048+0000 7efd2f7fe640 1 -- 172.21.15.78:0/1086448712 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7efd30126fa0 con 0x7efd3012cd90 2024-09-21T09:22:24.053 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:24.048+0000 7efd2effd640 1 --2- 172.21.15.78:0/1086448712 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7efd30129540 0x7efd301c4ef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:22:24.053 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:24.048+0000 7efd2f7fe640 1 --2- 172.21.15.78:0/1086448712 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7efd3012cd90 0x7efd3006f3e0 secure :-1 s=READY pgs=82 cs=0 l=1 rev1=1 crypto rx=0x7efd2800caa0 tx=0x7efd2800cf60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:24.053 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:24.048+0000 7efd177fe640 1 -- 172.21.15.78:0/1086448712 <== mon.0 v2:172.21.15.78:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7efd28016020 con 0x7efd3012cd90 2024-09-21T09:22:24.053 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:24.048+0000 7efd177fe640 1 -- 172.21.15.78:0/1086448712 <== mon.0 v2:172.21.15.78:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7efd2800ad10 con 0x7efd3012cd90 2024-09-21T09:22:24.053 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:24.048+0000 7efd355a1640 1 -- 172.21.15.78:0/1086448712 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7efd30116b80 con 0x7efd3012cd90 2024-09-21T09:22:24.053 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:24.048+0000 7efd177fe640 1 -- 172.21.15.78:0/1086448712 <== mon.0 v2:172.21.15.78:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7efd28012680 con 0x7efd3012cd90 2024-09-21T09:22:24.053 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:24.049+0000 7efd355a1640 1 -- 172.21.15.78:0/1086448712 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7efd301c5970 con 0x7efd3012cd90 2024-09-21T09:22:24.053 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:24.050+0000 7efd355a1640 1 -- 172.21.15.78:0/1086448712 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7efd30129920 con 0x7efd3012cd90 2024-09-21T09:22:24.054 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:24.050+0000 7efd177fe640 1 -- 172.21.15.78:0/1086448712 <== mon.0 v2:172.21.15.78:3300/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7efd28019020 con 0x7efd3012cd90 2024-09-21T09:22:24.054 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:24.050+0000 7efd177fe640 1 -- 172.21.15.78:0/1086448712 <== mon.0 v2:172.21.15.78:3300/0 5 ==== osd_map(24..24 src has 1..24) ==== 3027+0+0 (secure 0 0 0) 0x7efd280695e0 con 0x7efd3012cd90 2024-09-21T09:22:24.056 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:24.053+0000 7efd177fe640 1 -- 172.21.15.78:0/1086448712 <== mon.0 v2:172.21.15.78:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7efd28020020 con 0x7efd3012cd90 2024-09-21T09:22:24.208 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:24.206+0000 7efd355a1640 1 -- 172.21.15.78:0/1086448712 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7efd30069950 con 0x7efd3012cd90 2024-09-21T09:22:24.209 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:24.207+0000 7efd177fe640 1 -- 172.21.15.78:0/1086448712 <== mon.0 v2:172.21.15.78:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v24) ==== 74+0+8653 (secure 0 0 0) 0x7efd28068690 con 0x7efd3012cd90 2024-09-21T09:22:24.209 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:22:24.209 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":24,"fsid":"3633ad52-1063-4406-97d9-d53c339f43ca","created":"2024-09-21T09:20:38.248415+0000","modified":"2024-09-21T09:22:23.923196+0000","last_up_change":"2024-09-21T09:22:23.923196+0000","last_in_change":"2024-09-21T09:20:40.330925+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":4,"require_min_compat_client":"reef","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T09:20:51.169034+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-21T09:20:54.918007+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":17,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"6b18d222-0705-4495-a01c-3e2ba2ab4cff","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":0,"down_at":21,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6808","nonce":1449157914}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6809","nonce":1449157914}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6811","nonce":1449157914}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6810","nonce":1449157914}]},"public_addr":"172.21.15.78:6808/1449157914","cluster_addr":"172.21.15.78:6809/1449157914","heartbeat_back_addr":"172.21.15.78:6811/1449157914","heartbeat_front_addr":"172.21.15.78:6810/1449157914","state":["exists"]},{"osd":1,"uuid":"2d15b0fa-66c2-4bda-b64b-d72cbbb16fb1","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":22,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6800","nonce":3190796132}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6801","nonce":3190796132}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6805","nonce":3190796132}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6803","nonce":3190796132}]},"public_addr":"172.21.15.78:6800/3190796132","cluster_addr":"172.21.15.78:6801/3190796132","heartbeat_back_addr":"172.21.15.78:6805/3190796132","heartbeat_front_addr":"172.21.15.78:6803/3190796132","state":["exists"]},{"osd":2,"uuid":"0bf75d8f-017d-488d-8a2e-c3fa3b2a29c2","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":23,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6812","nonce":3442117044}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6813","nonce":3442117044}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6815","nonce":3442117044}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6814","nonce":3442117044}]},"public_addr":"172.21.15.78:6812/3442117044","cluster_addr":"172.21.15.78:6813/3442117044","heartbeat_back_addr":"172.21.15.78:6815/3442117044","heartbeat_front_addr":"172.21.15.78:6814/3442117044","state":["exists"]},{"osd":3,"uuid":"5e3788c8-b89a-4614-b3ec-7ee1250ddbe0","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":23,"down_at":24,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6802","nonce":4276488585}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6804","nonce":4276488585}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6807","nonce":4276488585}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6806","nonce":4276488585}]},"public_addr":"172.21.15.78:6802/4276488585","cluster_addr":"172.21.15.78:6804/4276488585","heartbeat_back_addr":"172.21.15.78:6807/4276488585","heartbeat_front_addr":"172.21.15.78:6806/4276488585","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:20:45.499319+0000","dead_epoch":21},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:20:45.489695+0000","dead_epoch":22},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:20:45.541959+0000","dead_epoch":23},{"osd":3,"down_stamp":"2024-09-21T09:22:23.923196+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:20:45.476915+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.78:6816/2602406893":"2024-09-22T09:22:16.387689+0000","172.21.15.78:0/1812672186":"2024-09-22T09:22:16.387689+0000","172.21.15.78:0/2894754949":"2024-09-22T09:22:16.387689+0000","172.21.15.78:0/2078474014":"2024-09-22T09:22:16.387689+0000","172.21.15.78:0/2453110290":"2024-09-22T09:22:16.387689+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T09:22:24.211 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:24.208+0000 7efd355a1640 1 -- 172.21.15.78:0/1086448712 >> v1:172.21.15.78:6816/2186157269 conn(0x7efd1c0514e0 legacy=0x7efd1c0539a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:24.211 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:24.208+0000 7efd355a1640 1 -- 172.21.15.78:0/1086448712 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7efd3012cd90 msgr2=0x7efd3006f3e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:24.211 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:24.208+0000 7efd355a1640 1 --2- 172.21.15.78:0/1086448712 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7efd3012cd90 0x7efd3006f3e0 secure :-1 s=READY pgs=82 cs=0 l=1 rev1=1 crypto rx=0x7efd2800caa0 tx=0x7efd2800cf60 comp rx=0 tx=0).stop 2024-09-21T09:22:24.212 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:24.208+0000 7efd355a1640 1 -- 172.21.15.78:0/1086448712 shutdown_connections 2024-09-21T09:22:24.212 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:24.208+0000 7efd355a1640 1 --2- 172.21.15.78:0/1086448712 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7efd3012cd90 0x7efd3006f3e0 unknown :-1 s=CLOSED pgs=82 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:24.212 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:24.208+0000 7efd355a1640 1 --2- 172.21.15.78:0/1086448712 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7efd30129e60 0x7efd301c5430 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:24.212 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:24.208+0000 7efd355a1640 1 --2- 172.21.15.78:0/1086448712 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7efd30129540 0x7efd301c4ef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:24.212 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:24.209+0000 7efd355a1640 1 -- 172.21.15.78:0/1086448712 >> 172.21.15.78:0/1086448712 conn(0x7efd3009d480 msgr2=0x7efd30134440 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:24.212 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:24.209+0000 7efd355a1640 1 -- 172.21.15.78:0/1086448712 shutdown_connections 2024-09-21T09:22:24.212 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:24.209+0000 7efd355a1640 1 -- 172.21.15.78:0/1086448712 wait complete. 2024-09-21T09:22:25.104 INFO:tasks.ceph.osd.3.smithi078.stderr:2024-09-21T09:22:25.101+0000 7fe153416640 -1 Falling back to public interface 2024-09-21T09:22:26.664 INFO:tasks.ceph.osd.0.smithi078.stderr:2024-09-21T09:22:26.662+0000 7fb036b34640 -1 osd.0 20 log_to_monitors true 2024-09-21T09:22:27.195 INFO:tasks.ceph.osd.1.smithi078.stderr:2024-09-21T09:22:27.192+0000 7f640d3dd640 -1 osd.1 20 log_to_monitors true 2024-09-21T09:22:27.222 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-09-21T09:22:27.323 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:27.320+0000 7f0b8b53f640 1 -- 172.21.15.78:0/1696644590 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f0b840991f0 msgr2=0x7f0b8412bcc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:27.323 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:27.320+0000 7f0b8b53f640 1 --2- 172.21.15.78:0/1696644590 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f0b840991f0 0x7f0b8412bcc0 secure :-1 s=READY pgs=32 cs=0 l=1 rev1=1 crypto rx=0x7f0b80009830 tx=0x7f0b8002a3f0 comp rx=0 tx=0).stop 2024-09-21T09:22:27.323 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:27.320+0000 7f0b8b53f640 1 -- 172.21.15.78:0/1696644590 shutdown_connections 2024-09-21T09:22:27.323 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:27.320+0000 7f0b8b53f640 1 --2- 172.21.15.78:0/1696644590 >> v2:172.21.15.78:3300/0 conn(0x7f0b8412c200 0x7f0b840956e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:27.323 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:27.320+0000 7f0b8b53f640 1 --2- 172.21.15.78:0/1696644590 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f0b840991f0 0x7f0b8412bcc0 unknown :-1 s=CLOSED pgs=32 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:27.323 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:27.320+0000 7f0b8b53f640 1 -- 172.21.15.78:0/1696644590 >> 172.21.15.78:0/1696644590 conn(0x7f0b8409a2e0 msgr2=0x7f0b84097ad0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:27.323 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:27.320+0000 7f0b8b53f640 1 -- 172.21.15.78:0/1696644590 shutdown_connections 2024-09-21T09:22:27.323 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:27.320+0000 7f0b8b53f640 1 -- 172.21.15.78:0/1696644590 wait complete. 2024-09-21T09:22:27.326 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:27.321+0000 7f0b8b53f640 1 Processor -- start 2024-09-21T09:22:27.326 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:27.321+0000 7f0b8b53f640 1 -- start start 2024-09-21T09:22:27.326 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:27.321+0000 7f0b8b53f640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f0b84095c20 0x7f0b8408c1f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:27.326 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:27.321+0000 7f0b8b53f640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f0b840991f0 0x7f0b8408c730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:27.326 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:27.321+0000 7f0b8b53f640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f0b8412c200 0x7f0b8408f350 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:27.326 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:27.321+0000 7f0b8b53f640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7f0b84134670 con 0x7f0b84095c20 2024-09-21T09:22:27.326 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:27.321+0000 7f0b8b53f640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7f0b841344f0 con 0x7f0b8412c200 2024-09-21T09:22:27.326 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:27.321+0000 7f0b8b53f640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7f0b841347f0 con 0x7f0b840991f0 2024-09-21T09:22:27.326 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:27.322+0000 7f0b892db640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f0b84095c20 0x7f0b8408c1f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:27.326 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:27.322+0000 7f0b88ada640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f0b840991f0 0x7f0b8408c730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:27.326 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:27.322+0000 7f0b89adc640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f0b8412c200 0x7f0b8408f350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:27.326 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:27.322+0000 7f0b88ada640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f0b840991f0 0x7f0b8408c730 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3302/0 says I am v2:172.21.15.78:45384/0 (socket says 172.21.15.78:45384) 2024-09-21T09:22:27.327 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:27.322+0000 7f0b88ada640 1 -- 172.21.15.78:0/3315238275 learned_addr learned my addr 172.21.15.78:0/3315238275 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:27.327 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:27.322+0000 7f0b88ada640 1 -- 172.21.15.78:0/3315238275 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f0b8412c200 msgr2=0x7f0b8408f350 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:27.327 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:27.322+0000 7f0b88ada640 1 --2- 172.21.15.78:0/3315238275 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f0b8412c200 0x7f0b8408f350 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:27.327 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:27.322+0000 7f0b88ada640 1 -- 172.21.15.78:0/3315238275 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f0b84095c20 msgr2=0x7f0b8408c1f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:27.327 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:27.322+0000 7f0b88ada640 1 --2- 172.21.15.78:0/3315238275 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f0b84095c20 0x7f0b8408c1f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:27.327 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:27.322+0000 7f0b88ada640 1 -- 172.21.15.78:0/3315238275 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f0b8408d690 con 0x7f0b840991f0 2024-09-21T09:22:27.327 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:27.322+0000 7f0b88ada640 1 --2- 172.21.15.78:0/3315238275 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f0b840991f0 0x7f0b8408c730 secure :-1 s=READY pgs=33 cs=0 l=1 rev1=1 crypto rx=0x7f0b7400e9c0 tx=0x7f0b7400ee80 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:27.327 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:27.322+0000 7f0b79ffb640 1 -- 172.21.15.78:0/3315238275 <== mon.2 v2:172.21.15.78:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0b7400cd80 con 0x7f0b840991f0 2024-09-21T09:22:27.327 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:27.322+0000 7f0b79ffb640 1 -- 172.21.15.78:0/3315238275 <== mon.2 v2:172.21.15.78:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f0b74004020 con 0x7f0b840991f0 2024-09-21T09:22:27.328 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:27.322+0000 7f0b79ffb640 1 -- 172.21.15.78:0/3315238275 <== mon.2 v2:172.21.15.78:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0b74004450 con 0x7f0b840991f0 2024-09-21T09:22:27.328 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:27.322+0000 7f0b8b53f640 1 -- 172.21.15.78:0/3315238275 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f0b84094030 con 0x7f0b840991f0 2024-09-21T09:22:27.328 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:27.322+0000 7f0b8b53f640 1 -- 172.21.15.78:0/3315238275 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f0b8408e5c0 con 0x7f0b840991f0 2024-09-21T09:22:27.328 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:27.323+0000 7f0b79ffb640 1 -- 172.21.15.78:0/3315238275 <== mon.2 v2:172.21.15.78:3302/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f0b740035f0 con 0x7f0b840991f0 2024-09-21T09:22:27.328 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:27.323+0000 7f0b8b53f640 1 -- 172.21.15.78:0/3315238275 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f0b8412bcc0 con 0x7f0b840991f0 2024-09-21T09:22:27.328 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:27.324+0000 7f0b79ffb640 1 -- 172.21.15.78:0/3315238275 <== mon.2 v2:172.21.15.78:3302/0 5 ==== osd_map(24..24 src has 1..24) ==== 3027+0+0 (secure 0 0 0) 0x7f0b740699a0 con 0x7f0b840991f0 2024-09-21T09:22:27.329 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:27.327+0000 7f0b79ffb640 1 -- 172.21.15.78:0/3315238275 <== mon.2 v2:172.21.15.78:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f0b74068a50 con 0x7f0b840991f0 2024-09-21T09:22:27.463 INFO:tasks.ceph.osd.0.smithi078.stderr:2024-09-21T09:22:27.460+0000 7fb02a8e3640 -1 osd.0 20 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:22:27.463 INFO:tasks.ceph.osd.1.smithi078.stderr:2024-09-21T09:22:27.461+0000 7f640118c640 -1 osd.1 20 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:22:27.485 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:27.483+0000 7f0b8b53f640 1 -- 172.21.15.78:0/3315238275 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7f0b840699c0 con 0x7f0b840991f0 2024-09-21T09:22:27.486 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:27.484+0000 7f0b79ffb640 1 -- 172.21.15.78:0/3315238275 <== mon.2 v2:172.21.15.78:3302/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v25) ==== 74+0+8653 (secure 0 0 0) 0x7f0b74009460 con 0x7f0b840991f0 2024-09-21T09:22:27.486 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:22:27.486 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":25,"fsid":"3633ad52-1063-4406-97d9-d53c339f43ca","created":"2024-09-21T09:20:38.248415+0000","modified":"2024-09-21T09:22:27.451998+0000","last_up_change":"2024-09-21T09:22:23.923196+0000","last_in_change":"2024-09-21T09:20:40.330925+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":4,"require_min_compat_client":"reef","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T09:20:51.169034+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-21T09:20:54.918007+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":17,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"6b18d222-0705-4495-a01c-3e2ba2ab4cff","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":0,"down_at":21,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6808","nonce":1449157914}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6809","nonce":1449157914}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6811","nonce":1449157914}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6810","nonce":1449157914}]},"public_addr":"172.21.15.78:6808/1449157914","cluster_addr":"172.21.15.78:6809/1449157914","heartbeat_back_addr":"172.21.15.78:6811/1449157914","heartbeat_front_addr":"172.21.15.78:6810/1449157914","state":["exists"]},{"osd":1,"uuid":"2d15b0fa-66c2-4bda-b64b-d72cbbb16fb1","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":22,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6800","nonce":3190796132}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6801","nonce":3190796132}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6805","nonce":3190796132}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6803","nonce":3190796132}]},"public_addr":"172.21.15.78:6800/3190796132","cluster_addr":"172.21.15.78:6801/3190796132","heartbeat_back_addr":"172.21.15.78:6805/3190796132","heartbeat_front_addr":"172.21.15.78:6803/3190796132","state":["exists"]},{"osd":2,"uuid":"0bf75d8f-017d-488d-8a2e-c3fa3b2a29c2","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":23,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6812","nonce":3442117044}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6813","nonce":3442117044}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6815","nonce":3442117044}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6814","nonce":3442117044}]},"public_addr":"172.21.15.78:6812/3442117044","cluster_addr":"172.21.15.78:6813/3442117044","heartbeat_back_addr":"172.21.15.78:6815/3442117044","heartbeat_front_addr":"172.21.15.78:6814/3442117044","state":["exists"]},{"osd":3,"uuid":"5e3788c8-b89a-4614-b3ec-7ee1250ddbe0","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":23,"down_at":24,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6802","nonce":4276488585}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6804","nonce":4276488585}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6807","nonce":4276488585}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6806","nonce":4276488585}]},"public_addr":"172.21.15.78:6802/4276488585","cluster_addr":"172.21.15.78:6804/4276488585","heartbeat_back_addr":"172.21.15.78:6807/4276488585","heartbeat_front_addr":"172.21.15.78:6806/4276488585","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:20:45.499319+0000","dead_epoch":21},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:20:45.489695+0000","dead_epoch":22},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:20:45.541959+0000","dead_epoch":23},{"osd":3,"down_stamp":"2024-09-21T09:22:23.923196+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:20:45.476915+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.78:6816/2602406893":"2024-09-22T09:22:16.387689+0000","172.21.15.78:0/1812672186":"2024-09-22T09:22:16.387689+0000","172.21.15.78:0/2894754949":"2024-09-22T09:22:16.387689+0000","172.21.15.78:0/2078474014":"2024-09-22T09:22:16.387689+0000","172.21.15.78:0/2453110290":"2024-09-22T09:22:16.387689+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T09:22:27.488 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:27.485+0000 7f0b8b53f640 1 -- 172.21.15.78:0/3315238275 >> v1:172.21.15.78:6816/2186157269 conn(0x7f0b680514b0 legacy=0x7f0b68053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:27.489 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:27.485+0000 7f0b8b53f640 1 -- 172.21.15.78:0/3315238275 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f0b840991f0 msgr2=0x7f0b8408c730 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:27.489 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:27.485+0000 7f0b8b53f640 1 --2- 172.21.15.78:0/3315238275 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f0b840991f0 0x7f0b8408c730 secure :-1 s=READY pgs=33 cs=0 l=1 rev1=1 crypto rx=0x7f0b7400e9c0 tx=0x7f0b7400ee80 comp rx=0 tx=0).stop 2024-09-21T09:22:27.489 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:27.486+0000 7f0b8b53f640 1 -- 172.21.15.78:0/3315238275 shutdown_connections 2024-09-21T09:22:27.489 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:27.486+0000 7f0b8b53f640 1 --2- 172.21.15.78:0/3315238275 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f0b8412c200 0x7f0b8408f350 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:27.489 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:27.486+0000 7f0b8b53f640 1 --2- 172.21.15.78:0/3315238275 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f0b840991f0 0x7f0b8408c730 unknown :-1 s=CLOSED pgs=33 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:27.489 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:27.486+0000 7f0b8b53f640 1 --2- 172.21.15.78:0/3315238275 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f0b84095c20 0x7f0b8408c1f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:27.489 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:27.486+0000 7f0b8b53f640 1 -- 172.21.15.78:0/3315238275 >> 172.21.15.78:0/3315238275 conn(0x7f0b8409a2e0 msgr2=0x7f0b84130b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:27.489 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:27.486+0000 7f0b8b53f640 1 -- 172.21.15.78:0/3315238275 shutdown_connections 2024-09-21T09:22:27.489 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:27.486+0000 7f0b8b53f640 1 -- 172.21.15.78:0/3315238275 wait complete. 2024-09-21T09:22:28.141 INFO:tasks.ceph.osd.2.smithi078.stderr:2024-09-21T09:22:28.138+0000 7f36f0f3e640 -1 osd.2 22 log_to_monitors true 2024-09-21T09:22:28.469 INFO:tasks.ceph.osd.2.smithi078.stderr:2024-09-21T09:22:28.467+0000 7f36e4ced640 -1 osd.2 22 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:22:29.391 INFO:tasks.ceph.osd.3.smithi078.stderr:2024-09-21T09:22:29.389+0000 7fe153416640 -1 osd.3 23 log_to_monitors true 2024-09-21T09:22:29.470 INFO:tasks.ceph.osd.3.smithi078.stderr:2024-09-21T09:22:29.468+0000 7fe146b99640 -1 osd.3 23 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:22:30.499 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-09-21T09:22:30.606 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.603+0000 7f0d61039640 1 -- 172.21.15.78:0/3322266996 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f0d5c132510 msgr2=0x7f0d5c134900 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:30.606 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.603+0000 7f0d61039640 1 --2- 172.21.15.78:0/3322266996 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f0d5c132510 0x7f0d5c134900 secure :-1 s=READY pgs=36 cs=0 l=1 rev1=1 crypto rx=0x7f0d4c009860 tx=0x7f0d4c02a460 comp rx=0 tx=0).stop 2024-09-21T09:22:30.606 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.603+0000 7f0d61039640 1 -- 172.21.15.78:0/3322266996 shutdown_connections 2024-09-21T09:22:30.606 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.603+0000 7f0d61039640 1 --2- 172.21.15.78:0/3322266996 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f0d5c12fbe0 0x7f0d5c131fd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:30.606 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.603+0000 7f0d61039640 1 --2- 172.21.15.78:0/3322266996 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f0d5c132510 0x7f0d5c134900 unknown :-1 s=CLOSED pgs=36 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:30.606 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.603+0000 7f0d61039640 1 -- 172.21.15.78:0/3322266996 >> 172.21.15.78:0/3322266996 conn(0x7f0d5c09a2e0 msgr2=0x7f0d5c097ad0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:30.606 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.603+0000 7f0d61039640 1 -- 172.21.15.78:0/3322266996 shutdown_connections 2024-09-21T09:22:30.606 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.603+0000 7f0d61039640 1 -- 172.21.15.78:0/3322266996 wait complete. 2024-09-21T09:22:30.608 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.604+0000 7f0d61039640 1 Processor -- start 2024-09-21T09:22:30.608 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.604+0000 7f0d61039640 1 -- start start 2024-09-21T09:22:30.608 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.605+0000 7f0d61039640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f0d5c12cc30 0x7f0d5c091100 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:30.609 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.605+0000 7f0d61039640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f0d5c12fbe0 0x7f0d5c091640 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:30.609 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.605+0000 7f0d61039640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f0d5c132510 0x7f0d5c091b80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:30.609 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.605+0000 7f0d61039640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7f0d5c1358b0 con 0x7f0d5c132510 2024-09-21T09:22:30.609 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.605+0000 7f0d61039640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7f0d5c135730 con 0x7f0d5c12cc30 2024-09-21T09:22:30.609 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.605+0000 7f0d61039640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7f0d5c135a30 con 0x7f0d5c12fbe0 2024-09-21T09:22:30.609 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.605+0000 7f0d5ad9d640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f0d5c12cc30 0x7f0d5c091100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:30.609 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.605+0000 7f0d5b59e640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f0d5c132510 0x7f0d5c091b80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:30.609 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.605+0000 7f0d5a59c640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f0d5c12fbe0 0x7f0d5c091640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:30.609 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.605+0000 7f0d5b59e640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f0d5c132510 0x7f0d5c091b80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3300/0 says I am v2:172.21.15.78:33860/0 (socket says 172.21.15.78:33860) 2024-09-21T09:22:30.609 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.605+0000 7f0d5b59e640 1 -- 172.21.15.78:0/3047911790 learned_addr learned my addr 172.21.15.78:0/3047911790 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:30.609 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.605+0000 7f0d5ad9d640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f0d5c12cc30 0x7f0d5c091100 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3301/0 says I am v2:172.21.15.78:44894/0 (socket says 172.21.15.78:44894) 2024-09-21T09:22:30.609 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.605+0000 7f0d5b59e640 1 -- 172.21.15.78:0/3047911790 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f0d5c12fbe0 msgr2=0x7f0d5c091640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:30.609 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.605+0000 7f0d5b59e640 1 --2- 172.21.15.78:0/3047911790 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f0d5c12fbe0 0x7f0d5c091640 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:30.609 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.605+0000 7f0d5b59e640 1 -- 172.21.15.78:0/3047911790 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f0d5c12cc30 msgr2=0x7f0d5c091100 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:30.609 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.605+0000 7f0d5b59e640 1 --2- 172.21.15.78:0/3047911790 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f0d5c12cc30 0x7f0d5c091100 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:30.610 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.605+0000 7f0d5b59e640 1 -- 172.21.15.78:0/3047911790 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f0d5c128240 con 0x7f0d5c132510 2024-09-21T09:22:30.610 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.605+0000 7f0d5ad9d640 1 --2- 172.21.15.78:0/3047911790 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f0d5c12cc30 0x7f0d5c091100 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:22:30.610 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.605+0000 7f0d5b59e640 1 --2- 172.21.15.78:0/3047911790 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f0d5c132510 0x7f0d5c091b80 secure :-1 s=READY pgs=89 cs=0 l=1 rev1=1 crypto rx=0x7f0d4c02aa50 tx=0x7f0d4c032b90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:30.610 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.606+0000 7f0d437fe640 1 -- 172.21.15.78:0/3047911790 <== mon.0 v2:172.21.15.78:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0d4c040070 con 0x7f0d5c132510 2024-09-21T09:22:30.610 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.606+0000 7f0d437fe640 1 -- 172.21.15.78:0/3047911790 <== mon.0 v2:172.21.15.78:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f0d4c005310 con 0x7f0d5c132510 2024-09-21T09:22:30.610 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.606+0000 7f0d61039640 1 -- 172.21.15.78:0/3047911790 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f0d5c12a0b0 con 0x7f0d5c132510 2024-09-21T09:22:30.610 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.606+0000 7f0d437fe640 1 -- 172.21.15.78:0/3047911790 <== mon.0 v2:172.21.15.78:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0d4c0047a0 con 0x7f0d5c132510 2024-09-21T09:22:30.610 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.606+0000 7f0d61039640 1 -- 172.21.15.78:0/3047911790 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f0d5c0920c0 con 0x7f0d5c132510 2024-09-21T09:22:30.611 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.607+0000 7f0d61039640 1 -- 172.21.15.78:0/3047911790 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f0d5c12a650 con 0x7f0d5c132510 2024-09-21T09:22:30.611 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.607+0000 7f0d437fe640 1 -- 172.21.15.78:0/3047911790 <== mon.0 v2:172.21.15.78:3300/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f0d4c039040 con 0x7f0d5c132510 2024-09-21T09:22:30.611 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.607+0000 7f0d437fe640 1 -- 172.21.15.78:0/3047911790 <== mon.0 v2:172.21.15.78:3300/0 5 ==== osd_map(28..28 src has 1..28) ==== 3027+0+0 (secure 0 0 0) 0x7f0d4c089f30 con 0x7f0d5c132510 2024-09-21T09:22:30.613 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.611+0000 7f0d437fe640 1 -- 172.21.15.78:0/3047911790 <== mon.0 v2:172.21.15.78:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f0d5c12a650 con 0x7f0d5c132510 2024-09-21T09:22:30.763 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.761+0000 7f0d61039640 1 -- 172.21.15.78:0/3047911790 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7f0d5c12d010 con 0x7f0d5c132510 2024-09-21T09:22:30.764 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.761+0000 7f0d437fe640 1 -- 172.21.15.78:0/3047911790 <== mon.0 v2:172.21.15.78:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v28) ==== 74+0+9174 (secure 0 0 0) 0x7f0d5c128240 con 0x7f0d5c132510 2024-09-21T09:22:30.764 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:22:30.764 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":28,"fsid":"3633ad52-1063-4406-97d9-d53c339f43ca","created":"2024-09-21T09:20:38.248415+0000","modified":"2024-09-21T09:22:30.459558+0000","last_up_change":"2024-09-21T09:22:30.459558+0000","last_in_change":"2024-09-21T09:20:40.330925+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":7,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":4,"require_min_compat_client":"reef","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T09:20:51.169034+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-21T09:20:54.918007+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":17,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"6b18d222-0705-4495-a01c-3e2ba2ab4cff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":9,"last_clean_end":20,"up_from":26,"up_thru":26,"down_at":21,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6800","nonce":2538472627}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6801","nonce":2538472627}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6805","nonce":2538472627}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6803","nonce":2538472627}]},"public_addr":"172.21.15.78:6800/2538472627","cluster_addr":"172.21.15.78:6801/2538472627","heartbeat_back_addr":"172.21.15.78:6805/2538472627","heartbeat_front_addr":"172.21.15.78:6803/2538472627","state":["exists","up"]},{"osd":1,"uuid":"2d15b0fa-66c2-4bda-b64b-d72cbbb16fb1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":9,"last_clean_end":20,"up_from":26,"up_thru":26,"down_at":22,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6802","nonce":3598780309}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6804","nonce":3598780309}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6807","nonce":3598780309}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6806","nonce":3598780309}]},"public_addr":"172.21.15.78:6802/3598780309","cluster_addr":"172.21.15.78:6804/3598780309","heartbeat_back_addr":"172.21.15.78:6807/3598780309","heartbeat_front_addr":"172.21.15.78:6806/3598780309","state":["exists","up"]},{"osd":2,"uuid":"0bf75d8f-017d-488d-8a2e-c3fa3b2a29c2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":9,"last_clean_end":22,"up_from":27,"up_thru":27,"down_at":23,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6808","nonce":2733362473}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6809","nonce":2733362473}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6811","nonce":2733362473}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6810","nonce":2733362473}]},"public_addr":"172.21.15.78:6808/2733362473","cluster_addr":"172.21.15.78:6809/2733362473","heartbeat_back_addr":"172.21.15.78:6811/2733362473","heartbeat_front_addr":"172.21.15.78:6810/2733362473","state":["exists","up"]},{"osd":3,"uuid":"5e3788c8-b89a-4614-b3ec-7ee1250ddbe0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":9,"last_clean_end":23,"up_from":28,"up_thru":23,"down_at":24,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6812","nonce":3499964365}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6813","nonce":3499964365}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6815","nonce":3499964365}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6814","nonce":3499964365}]},"public_addr":"172.21.15.78:6812/3499964365","cluster_addr":"172.21.15.78:6813/3499964365","heartbeat_back_addr":"172.21.15.78:6815/3499964365","heartbeat_front_addr":"172.21.15.78:6814/3499964365","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:20:45.499319+0000","dead_epoch":21},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:20:45.489695+0000","dead_epoch":22},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:20:45.541959+0000","dead_epoch":23},{"osd":3,"down_stamp":"2024-09-21T09:22:23.923196+0000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:20:45.476915+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.78:6816/2602406893":"2024-09-22T09:22:16.387689+0000","172.21.15.78:0/1812672186":"2024-09-22T09:22:16.387689+0000","172.21.15.78:0/2894754949":"2024-09-22T09:22:16.387689+0000","172.21.15.78:0/2078474014":"2024-09-22T09:22:16.387689+0000","172.21.15.78:0/2453110290":"2024-09-22T09:22:16.387689+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T09:22:30.766 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.763+0000 7f0d61039640 1 -- 172.21.15.78:0/3047911790 >> v1:172.21.15.78:6816/2186157269 conn(0x7f0d480514e0 legacy=0x7f0d480539a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:30.766 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.763+0000 7f0d61039640 1 -- 172.21.15.78:0/3047911790 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f0d5c132510 msgr2=0x7f0d5c091b80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:30.766 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.763+0000 7f0d61039640 1 --2- 172.21.15.78:0/3047911790 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f0d5c132510 0x7f0d5c091b80 secure :-1 s=READY pgs=89 cs=0 l=1 rev1=1 crypto rx=0x7f0d4c02aa50 tx=0x7f0d4c032b90 comp rx=0 tx=0).stop 2024-09-21T09:22:30.766 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.763+0000 7f0d61039640 1 -- 172.21.15.78:0/3047911790 shutdown_connections 2024-09-21T09:22:30.766 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.763+0000 7f0d61039640 1 --2- 172.21.15.78:0/3047911790 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f0d5c132510 0x7f0d5c091b80 unknown :-1 s=CLOSED pgs=89 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:30.766 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.763+0000 7f0d61039640 1 --2- 172.21.15.78:0/3047911790 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f0d5c12fbe0 0x7f0d5c091640 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:30.766 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.763+0000 7f0d61039640 1 --2- 172.21.15.78:0/3047911790 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f0d5c12cc30 0x7f0d5c091100 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:30.766 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.763+0000 7f0d61039640 1 -- 172.21.15.78:0/3047911790 >> 172.21.15.78:0/3047911790 conn(0x7f0d5c09a2e0 msgr2=0x7f0d5c097ad0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:30.766 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.763+0000 7f0d61039640 1 -- 172.21.15.78:0/3047911790 shutdown_connections 2024-09-21T09:22:30.766 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.763+0000 7f0d61039640 1 -- 172.21.15.78:0/3047911790 wait complete. 2024-09-21T09:22:30.776 INFO:tasks.ceph.ceph_manager.ceph:all up! 2024-09-21T09:22:30.776 INFO:teuthology.run_tasks:Running task exec... 2024-09-21T09:22:30.793 INFO:teuthology.task.exec:Executing custom commands... 2024-09-21T09:22:30.793 INFO:teuthology.task.exec:Running commands on role mon.a host ubuntu@smithi078.front.sepia.ceph.com 2024-09-21T09:22:30.793 DEBUG:teuthology.orchestra.run.smithi078:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph versions' 2024-09-21T09:22:30.889 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.887+0000 7f60d34c4640 1 -- 172.21.15.78:0/3080000964 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f60cc072f50 msgr2=0x7f60cc073330 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:30.890 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.887+0000 7f60d34c4640 1 --2- 172.21.15.78:0/3080000964 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f60cc072f50 0x7f60cc073330 secure :-1 s=READY pgs=91 cs=0 l=1 rev1=1 crypto rx=0x7f60c4009830 tx=0x7f60c402a3f0 comp rx=0 tx=0).stop 2024-09-21T09:22:30.890 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.887+0000 7f60d34c4640 1 -- 172.21.15.78:0/3080000964 shutdown_connections 2024-09-21T09:22:30.890 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.887+0000 7f60d34c4640 1 --2- 172.21.15.78:0/3080000964 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f60cc12d030 0x7f60cc134ca0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:30.890 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.887+0000 7f60d34c4640 1 --2- 172.21.15.78:0/3080000964 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f60cc072f50 0x7f60cc073330 unknown :-1 s=CLOSED pgs=91 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:30.890 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.887+0000 7f60d34c4640 1 -- 172.21.15.78:0/3080000964 >> 172.21.15.78:0/3080000964 conn(0x7f60cc09d5d0 msgr2=0x7f60cc09d9e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:30.890 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.887+0000 7f60d34c4640 1 -- 172.21.15.78:0/3080000964 shutdown_connections 2024-09-21T09:22:30.890 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.887+0000 7f60d34c4640 1 -- 172.21.15.78:0/3080000964 wait complete. 2024-09-21T09:22:30.892 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.888+0000 7f60d34c4640 1 Processor -- start 2024-09-21T09:22:30.892 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.888+0000 7f60d34c4640 1 -- start start 2024-09-21T09:22:30.893 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.888+0000 7f60d34c4640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f60cc072f50 0x7f60cc12da30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:30.893 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.888+0000 7f60d34c4640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f60cc073870 0x7f60cc12df70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:30.893 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.888+0000 7f60d34c4640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f60cc12d030 0x7f60cc12e4b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:30.893 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.888+0000 7f60d34c4640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7f60cc135840 con 0x7f60cc073870 2024-09-21T09:22:30.893 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.888+0000 7f60d34c4640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7f60cc1356c0 con 0x7f60cc072f50 2024-09-21T09:22:30.893 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.888+0000 7f60d34c4640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7f60cc1359c0 con 0x7f60cc12d030 2024-09-21T09:22:30.893 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.888+0000 7f60d1260640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f60cc072f50 0x7f60cc12da30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:30.893 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.888+0000 7f60d1260640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f60cc072f50 0x7f60cc12da30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3301/0 says I am v2:172.21.15.78:44924/0 (socket says 172.21.15.78:44924) 2024-09-21T09:22:30.893 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.888+0000 7f60d1260640 1 -- 172.21.15.78:0/3740512040 learned_addr learned my addr 172.21.15.78:0/3740512040 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:30.893 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.888+0000 7f60d0a5f640 1 --2- 172.21.15.78:0/3740512040 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f60cc073870 0x7f60cc12df70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:30.893 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.888+0000 7f60d1a61640 1 --2- 172.21.15.78:0/3740512040 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f60cc12d030 0x7f60cc12e4b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:30.893 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.889+0000 7f60d0a5f640 1 -- 172.21.15.78:0/3740512040 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f60cc12d030 msgr2=0x7f60cc12e4b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:30.893 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.889+0000 7f60d0a5f640 1 --2- 172.21.15.78:0/3740512040 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f60cc12d030 0x7f60cc12e4b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:30.893 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.889+0000 7f60d0a5f640 1 -- 172.21.15.78:0/3740512040 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f60cc072f50 msgr2=0x7f60cc12da30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:30.894 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.889+0000 7f60d0a5f640 1 --2- 172.21.15.78:0/3740512040 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f60cc072f50 0x7f60cc12da30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:30.894 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.889+0000 7f60d0a5f640 1 -- 172.21.15.78:0/3740512040 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f60cc1295b0 con 0x7f60cc073870 2024-09-21T09:22:30.894 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.889+0000 7f60d1260640 1 --2- 172.21.15.78:0/3740512040 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f60cc072f50 0x7f60cc12da30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:22:30.894 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.889+0000 7f60d0a5f640 1 --2- 172.21.15.78:0/3740512040 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f60cc073870 0x7f60cc12df70 secure :-1 s=READY pgs=92 cs=0 l=1 rev1=1 crypto rx=0x7f60c000cb40 tx=0x7f60c000cf40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:30.894 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.889+0000 7f60c9ffb640 1 -- 172.21.15.78:0/3740512040 <== mon.0 v2:172.21.15.78:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f60c0016020 con 0x7f60cc073870 2024-09-21T09:22:30.894 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.889+0000 7f60c9ffb640 1 -- 172.21.15.78:0/3740512040 <== mon.0 v2:172.21.15.78:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f60c000ad20 con 0x7f60cc073870 2024-09-21T09:22:30.894 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.889+0000 7f60d34c4640 1 -- 172.21.15.78:0/3740512040 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f60cc12e9f0 con 0x7f60cc073870 2024-09-21T09:22:30.894 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.889+0000 7f60c9ffb640 1 -- 172.21.15.78:0/3740512040 <== mon.0 v2:172.21.15.78:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f60c0012680 con 0x7f60cc073870 2024-09-21T09:22:30.894 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.889+0000 7f60d34c4640 1 -- 172.21.15.78:0/3740512040 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f60cc12f3d0 con 0x7f60cc073870 2024-09-21T09:22:30.894 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.890+0000 7f60d34c4640 1 -- 172.21.15.78:0/3740512040 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f60cc073330 con 0x7f60cc073870 2024-09-21T09:22:30.894 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.890+0000 7f60c9ffb640 1 -- 172.21.15.78:0/3740512040 <== mon.0 v2:172.21.15.78:3300/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f60c0019020 con 0x7f60cc073870 2024-09-21T09:22:30.895 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.891+0000 7f60c9ffb640 1 -- 172.21.15.78:0/3740512040 <== mon.0 v2:172.21.15.78:3300/0 5 ==== osd_map(28..28 src has 1..28) ==== 3027+0+0 (secure 0 0 0) 0x7f60c0069220 con 0x7f60cc073870 2024-09-21T09:22:30.896 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:30.894+0000 7f60c9ffb640 1 -- 172.21.15.78:0/3740512040 <== mon.0 v2:172.21.15.78:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f60c0020030 con 0x7f60cc073870 2024-09-21T09:22:31.099 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.097+0000 7f60d34c4640 1 -- 172.21.15.78:0/3740512040 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "versions"} v 0) -- 0x7f60cc0689e0 con 0x7f60cc073870 2024-09-21T09:22:31.100 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.098+0000 7f60c9ffb640 1 -- 172.21.15.78:0/3740512040 <== mon.0 v2:172.21.15.78:3300/0 7 ==== mon_command_ack([{"prefix": "versions"}]=0 v0) ==== 56+0+499 (secure 0 0 0) 0x7f60c00682d0 con 0x7f60cc073870 2024-09-21T09:22:31.101 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-21T09:22:31.101 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-21T09:22:31.101 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 3 2024-09-21T09:22:31.101 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T09:22:31.101 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-21T09:22:31.101 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:22:31.102 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T09:22:31.102 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-21T09:22:31.102 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 4 2024-09-21T09:22:31.102 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T09:22:31.102 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-21T09:22:31.102 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 9 2024-09-21T09:22:31.102 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-21T09:22:31.103 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-21T09:22:31.104 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.099+0000 7f60d34c4640 1 -- 172.21.15.78:0/3740512040 >> v1:172.21.15.78:6816/2186157269 conn(0x7f60b80514b0 legacy=0x7f60b8053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:31.104 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.099+0000 7f60d34c4640 1 -- 172.21.15.78:0/3740512040 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f60cc073870 msgr2=0x7f60cc12df70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:31.104 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.099+0000 7f60d34c4640 1 --2- 172.21.15.78:0/3740512040 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f60cc073870 0x7f60cc12df70 secure :-1 s=READY pgs=92 cs=0 l=1 rev1=1 crypto rx=0x7f60c000cb40 tx=0x7f60c000cf40 comp rx=0 tx=0).stop 2024-09-21T09:22:31.104 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.100+0000 7f60d34c4640 1 -- 172.21.15.78:0/3740512040 shutdown_connections 2024-09-21T09:22:31.104 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.100+0000 7f60d34c4640 1 --2- 172.21.15.78:0/3740512040 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f60cc12d030 0x7f60cc12e4b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:31.104 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.100+0000 7f60d34c4640 1 --2- 172.21.15.78:0/3740512040 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f60cc073870 0x7f60cc12df70 unknown :-1 s=CLOSED pgs=92 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:31.104 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.100+0000 7f60d34c4640 1 --2- 172.21.15.78:0/3740512040 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f60cc072f50 0x7f60cc12da30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:31.104 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.100+0000 7f60d34c4640 1 -- 172.21.15.78:0/3740512040 >> 172.21.15.78:0/3740512040 conn(0x7f60cc09d5d0 msgr2=0x7f60cc08f7d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:31.104 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.100+0000 7f60d34c4640 1 -- 172.21.15.78:0/3740512040 shutdown_connections 2024-09-21T09:22:31.105 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.100+0000 7f60d34c4640 1 -- 172.21.15.78:0/3740512040 wait complete. 2024-09-21T09:22:31.112 DEBUG:teuthology.orchestra.run.smithi078:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph osd dump -f json-pretty' 2024-09-21T09:22:31.208 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.205+0000 7f136d17d640 1 -- 172.21.15.78:0/1302003675 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f1368097500 msgr2=0x7f1368131cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:31.208 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.205+0000 7f136d17d640 1 --2- 172.21.15.78:0/1302003675 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f1368097500 0x7f1368131cc0 secure :-1 s=READY pgs=41 cs=0 l=1 rev1=1 crypto rx=0x7f135c005fd0 tx=0x7f135c02a4b0 comp rx=0 tx=0).stop 2024-09-21T09:22:31.208 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.205+0000 7f136d17d640 1 -- 172.21.15.78:0/1302003675 shutdown_connections 2024-09-21T09:22:31.208 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.205+0000 7f136d17d640 1 --2- 172.21.15.78:0/1302003675 >> v2:172.21.15.78:3300/0 conn(0x7f1368132200 0x7f13681345f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:31.209 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.205+0000 7f136d17d640 1 --2- 172.21.15.78:0/1302003675 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f1368097500 0x7f1368131cc0 unknown :-1 s=CLOSED pgs=41 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:31.209 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.205+0000 7f136d17d640 1 -- 172.21.15.78:0/1302003675 >> 172.21.15.78:0/1302003675 conn(0x7f136809b580 msgr2=0x7f136809b990 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:31.209 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.205+0000 7f136d17d640 1 -- 172.21.15.78:0/1302003675 shutdown_connections 2024-09-21T09:22:31.209 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.205+0000 7f136d17d640 1 -- 172.21.15.78:0/1302003675 wait complete. 2024-09-21T09:22:31.210 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.207+0000 7f136d17d640 1 Processor -- start 2024-09-21T09:22:31.211 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.207+0000 7f136d17d640 1 -- start start 2024-09-21T09:22:31.211 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.207+0000 7f136d17d640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f1368097500 0x7f1368094ba0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:31.211 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.207+0000 7f136d17d640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f136812cf60 0x7f13680950e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:31.211 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.207+0000 7f136d17d640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f1368132200 0x7f1368095620 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:31.211 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.207+0000 7f136d17d640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7f1368135a60 con 0x7f1368132200 2024-09-21T09:22:31.211 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.207+0000 7f136d17d640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7f13681358e0 con 0x7f136812cf60 2024-09-21T09:22:31.211 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.207+0000 7f136d17d640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7f1368135be0 con 0x7f1368097500 2024-09-21T09:22:31.211 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.207+0000 7f136659c640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f136812cf60 0x7f13680950e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:31.211 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.207+0000 7f136759e640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f1368132200 0x7f1368095620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:31.211 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.207+0000 7f136659c640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f136812cf60 0x7f13680950e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3301/0 says I am v2:172.21.15.78:44944/0 (socket says 172.21.15.78:44944) 2024-09-21T09:22:31.211 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.207+0000 7f1366d9d640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f1368097500 0x7f1368094ba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:31.211 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.207+0000 7f136659c640 1 -- 172.21.15.78:0/786850095 learned_addr learned my addr 172.21.15.78:0/786850095 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:31.211 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.207+0000 7f136659c640 1 -- 172.21.15.78:0/786850095 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f1368097500 msgr2=0x7f1368094ba0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:31.212 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.207+0000 7f136659c640 1 --2- 172.21.15.78:0/786850095 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f1368097500 0x7f1368094ba0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:31.212 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.207+0000 7f136659c640 1 -- 172.21.15.78:0/786850095 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f1368132200 msgr2=0x7f1368095620 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:31.212 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.207+0000 7f136659c640 1 --2- 172.21.15.78:0/786850095 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f1368132200 0x7f1368095620 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:31.212 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.207+0000 7f136659c640 1 -- 172.21.15.78:0/786850095 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f136806ef00 con 0x7f136812cf60 2024-09-21T09:22:31.212 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.207+0000 7f136759e640 1 --2- 172.21.15.78:0/786850095 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f1368132200 0x7f1368095620 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:22:31.212 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.208+0000 7f136659c640 1 --2- 172.21.15.78:0/786850095 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f136812cf60 0x7f13680950e0 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7f135c000c00 tx=0x7f135c0038a0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:31.212 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.208+0000 7f134f7fe640 1 -- 172.21.15.78:0/786850095 <== mon.1 v2:172.21.15.78:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f135c03f070 con 0x7f136812cf60 2024-09-21T09:22:31.212 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.208+0000 7f134f7fe640 1 -- 172.21.15.78:0/786850095 <== mon.1 v2:172.21.15.78:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f135c0047a0 con 0x7f136812cf60 2024-09-21T09:22:31.212 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.208+0000 7f134f7fe640 1 -- 172.21.15.78:0/786850095 <== mon.1 v2:172.21.15.78:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f135c035030 con 0x7f136812cf60 2024-09-21T09:22:31.212 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.208+0000 7f136d17d640 1 -- 172.21.15.78:0/786850095 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f136806f6e0 con 0x7f136812cf60 2024-09-21T09:22:31.212 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.208+0000 7f136d17d640 1 -- 172.21.15.78:0/786850095 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7f13681cae70 con 0x7f136812cf60 2024-09-21T09:22:31.213 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.209+0000 7f136d17d640 1 -- 172.21.15.78:0/786850095 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f136812d340 con 0x7f136812cf60 2024-09-21T09:22:31.213 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.209+0000 7f134f7fe640 1 -- 172.21.15.78:0/786850095 <== mon.1 v2:172.21.15.78:3301/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f135c042070 con 0x7f136812cf60 2024-09-21T09:22:31.213 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.209+0000 7f134f7fe640 1 -- 172.21.15.78:0/786850095 <== mon.1 v2:172.21.15.78:3301/0 5 ==== osd_map(28..28 src has 1..28) ==== 3027+0+0 (secure 0 0 0) 0x7f135c089ff0 con 0x7f136812cf60 2024-09-21T09:22:31.215 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.213+0000 7f134f7fe640 1 -- 172.21.15.78:0/786850095 <== mon.1 v2:172.21.15.78:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f136806ef00 con 0x7f136812cf60 2024-09-21T09:22:31.363 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.360+0000 7f136d17d640 1 -- 172.21.15.78:0/786850095 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_command({"prefix": "osd dump", "format": "json-pretty"} v 0) -- 0x7f1368074a10 con 0x7f136812cf60 2024-09-21T09:22:31.363 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.361+0000 7f134f7fe640 1 -- 172.21.15.78:0/786850095 <== mon.1 v2:172.21.15.78:3301/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json-pretty"}]=0 v28) ==== 81+0+16838 (secure 0 0 0) 0x7f135c0890a0 con 0x7f136812cf60 2024-09-21T09:22:31.364 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:22:31.364 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-21T09:22:31.364 INFO:teuthology.orchestra.run.smithi078.stdout: "epoch": 28, 2024-09-21T09:22:31.364 INFO:teuthology.orchestra.run.smithi078.stdout: "fsid": "3633ad52-1063-4406-97d9-d53c339f43ca", 2024-09-21T09:22:31.364 INFO:teuthology.orchestra.run.smithi078.stdout: "created": "2024-09-21T09:20:38.248415+0000", 2024-09-21T09:22:31.364 INFO:teuthology.orchestra.run.smithi078.stdout: "modified": "2024-09-21T09:22:30.459558+0000", 2024-09-21T09:22:31.364 INFO:teuthology.orchestra.run.smithi078.stdout: "last_up_change": "2024-09-21T09:22:30.459558+0000", 2024-09-21T09:22:31.364 INFO:teuthology.orchestra.run.smithi078.stdout: "last_in_change": "2024-09-21T09:20:40.330925+0000", 2024-09-21T09:22:31.364 INFO:teuthology.orchestra.run.smithi078.stdout: "flags": "sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit", 2024-09-21T09:22:31.364 INFO:teuthology.orchestra.run.smithi078.stdout: "flags_num": 5799936, 2024-09-21T09:22:31.365 INFO:teuthology.orchestra.run.smithi078.stdout: "flags_set": [ 2024-09-21T09:22:31.365 INFO:teuthology.orchestra.run.smithi078.stdout: "pglog_hardlimit", 2024-09-21T09:22:31.365 INFO:teuthology.orchestra.run.smithi078.stdout: "purged_snapdirs", 2024-09-21T09:22:31.365 INFO:teuthology.orchestra.run.smithi078.stdout: "recovery_deletes", 2024-09-21T09:22:31.365 INFO:teuthology.orchestra.run.smithi078.stdout: "sortbitwise" 2024-09-21T09:22:31.365 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-09-21T09:22:31.365 INFO:teuthology.orchestra.run.smithi078.stdout: "crush_version": 7, 2024-09-21T09:22:31.365 INFO:teuthology.orchestra.run.smithi078.stdout: "full_ratio": 0.94999998807907104, 2024-09-21T09:22:31.365 INFO:teuthology.orchestra.run.smithi078.stdout: "backfillfull_ratio": 0.89999997615814209, 2024-09-21T09:22:31.365 INFO:teuthology.orchestra.run.smithi078.stdout: "nearfull_ratio": 0.85000002384185791, 2024-09-21T09:22:31.365 INFO:teuthology.orchestra.run.smithi078.stdout: "cluster_snapshot": "", 2024-09-21T09:22:31.365 INFO:teuthology.orchestra.run.smithi078.stdout: "pool_max": 2, 2024-09-21T09:22:31.365 INFO:teuthology.orchestra.run.smithi078.stdout: "max_osd": 4, 2024-09-21T09:22:31.366 INFO:teuthology.orchestra.run.smithi078.stdout: "require_min_compat_client": "reef", 2024-09-21T09:22:31.366 INFO:teuthology.orchestra.run.smithi078.stdout: "min_compat_client": "jewel", 2024-09-21T09:22:31.366 INFO:teuthology.orchestra.run.smithi078.stdout: "require_osd_release": "reef", 2024-09-21T09:22:31.366 INFO:teuthology.orchestra.run.smithi078.stdout: "allow_crimson": false, 2024-09-21T09:22:31.366 INFO:teuthology.orchestra.run.smithi078.stdout: "pools": [ 2024-09-21T09:22:31.366 INFO:teuthology.orchestra.run.smithi078.stdout: { 2024-09-21T09:22:31.366 INFO:teuthology.orchestra.run.smithi078.stdout: "pool": 1, 2024-09-21T09:22:31.366 INFO:teuthology.orchestra.run.smithi078.stdout: "pool_name": ".mgr", 2024-09-21T09:22:31.366 INFO:teuthology.orchestra.run.smithi078.stdout: "create_time": "2024-09-21T09:20:51.169034+0000", 2024-09-21T09:22:31.366 INFO:teuthology.orchestra.run.smithi078.stdout: "flags": 1, 2024-09-21T09:22:31.366 INFO:teuthology.orchestra.run.smithi078.stdout: "flags_names": "hashpspool", 2024-09-21T09:22:31.366 INFO:teuthology.orchestra.run.smithi078.stdout: "type": 1, 2024-09-21T09:22:31.367 INFO:teuthology.orchestra.run.smithi078.stdout: "size": 2, 2024-09-21T09:22:31.367 INFO:teuthology.orchestra.run.smithi078.stdout: "min_size": 1, 2024-09-21T09:22:31.367 INFO:teuthology.orchestra.run.smithi078.stdout: "crush_rule": 0, 2024-09-21T09:22:31.367 INFO:teuthology.orchestra.run.smithi078.stdout: "peering_crush_bucket_count": 0, 2024-09-21T09:22:31.367 INFO:teuthology.orchestra.run.smithi078.stdout: "peering_crush_bucket_target": 0, 2024-09-21T09:22:31.367 INFO:teuthology.orchestra.run.smithi078.stdout: "peering_crush_bucket_barrier": 0, 2024-09-21T09:22:31.367 INFO:teuthology.orchestra.run.smithi078.stdout: "peering_crush_bucket_mandatory_member": 2147483647, 2024-09-21T09:22:31.367 INFO:teuthology.orchestra.run.smithi078.stdout: "is_stretch_pool": false, 2024-09-21T09:22:31.367 INFO:teuthology.orchestra.run.smithi078.stdout: "object_hash": 2, 2024-09-21T09:22:31.367 INFO:teuthology.orchestra.run.smithi078.stdout: "pg_autoscale_mode": "off", 2024-09-21T09:22:31.367 INFO:teuthology.orchestra.run.smithi078.stdout: "pg_num": 1, 2024-09-21T09:22:31.367 INFO:teuthology.orchestra.run.smithi078.stdout: "pg_placement_num": 1, 2024-09-21T09:22:31.367 INFO:teuthology.orchestra.run.smithi078.stdout: "pg_placement_num_target": 1, 2024-09-21T09:22:31.368 INFO:teuthology.orchestra.run.smithi078.stdout: "pg_num_target": 1, 2024-09-21T09:22:31.368 INFO:teuthology.orchestra.run.smithi078.stdout: "pg_num_pending": 1, 2024-09-21T09:22:31.368 INFO:teuthology.orchestra.run.smithi078.stdout: "last_pg_merge_meta": { 2024-09-21T09:22:31.368 INFO:teuthology.orchestra.run.smithi078.stdout: "source_pgid": "0.0", 2024-09-21T09:22:31.368 INFO:teuthology.orchestra.run.smithi078.stdout: "ready_epoch": 0, 2024-09-21T09:22:31.368 INFO:teuthology.orchestra.run.smithi078.stdout: "last_epoch_started": 0, 2024-09-21T09:22:31.368 INFO:teuthology.orchestra.run.smithi078.stdout: "last_epoch_clean": 0, 2024-09-21T09:22:31.368 INFO:teuthology.orchestra.run.smithi078.stdout: "source_version": "0'0", 2024-09-21T09:22:31.368 INFO:teuthology.orchestra.run.smithi078.stdout: "target_version": "0'0" 2024-09-21T09:22:31.368 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T09:22:31.368 INFO:teuthology.orchestra.run.smithi078.stdout: "last_change": "13", 2024-09-21T09:22:31.368 INFO:teuthology.orchestra.run.smithi078.stdout: "last_force_op_resend": "0", 2024-09-21T09:22:31.368 INFO:teuthology.orchestra.run.smithi078.stdout: "last_force_op_resend_prenautilus": "0", 2024-09-21T09:22:31.368 INFO:teuthology.orchestra.run.smithi078.stdout: "last_force_op_resend_preluminous": "0", 2024-09-21T09:22:31.369 INFO:teuthology.orchestra.run.smithi078.stdout: "auid": 0, 2024-09-21T09:22:31.369 INFO:teuthology.orchestra.run.smithi078.stdout: "snap_mode": "selfmanaged", 2024-09-21T09:22:31.369 INFO:teuthology.orchestra.run.smithi078.stdout: "snap_seq": 0, 2024-09-21T09:22:31.369 INFO:teuthology.orchestra.run.smithi078.stdout: "snap_epoch": 0, 2024-09-21T09:22:31.369 INFO:teuthology.orchestra.run.smithi078.stdout: "pool_snaps": [], 2024-09-21T09:22:31.369 INFO:teuthology.orchestra.run.smithi078.stdout: "removed_snaps": "[]", 2024-09-21T09:22:31.369 INFO:teuthology.orchestra.run.smithi078.stdout: "quota_max_bytes": 0, 2024-09-21T09:22:31.369 INFO:teuthology.orchestra.run.smithi078.stdout: "quota_max_objects": 0, 2024-09-21T09:22:31.369 INFO:teuthology.orchestra.run.smithi078.stdout: "tiers": [], 2024-09-21T09:22:31.369 INFO:teuthology.orchestra.run.smithi078.stdout: "tier_of": -1, 2024-09-21T09:22:31.369 INFO:teuthology.orchestra.run.smithi078.stdout: "read_tier": -1, 2024-09-21T09:22:31.369 INFO:teuthology.orchestra.run.smithi078.stdout: "write_tier": -1, 2024-09-21T09:22:31.369 INFO:teuthology.orchestra.run.smithi078.stdout: "cache_mode": "none", 2024-09-21T09:22:31.370 INFO:teuthology.orchestra.run.smithi078.stdout: "target_max_bytes": 0, 2024-09-21T09:22:31.370 INFO:teuthology.orchestra.run.smithi078.stdout: "target_max_objects": 0, 2024-09-21T09:22:31.370 INFO:teuthology.orchestra.run.smithi078.stdout: "cache_target_dirty_ratio_micro": 400000, 2024-09-21T09:22:31.370 INFO:teuthology.orchestra.run.smithi078.stdout: "cache_target_dirty_high_ratio_micro": 600000, 2024-09-21T09:22:31.370 INFO:teuthology.orchestra.run.smithi078.stdout: "cache_target_full_ratio_micro": 800000, 2024-09-21T09:22:31.370 INFO:teuthology.orchestra.run.smithi078.stdout: "cache_min_flush_age": 0, 2024-09-21T09:22:31.370 INFO:teuthology.orchestra.run.smithi078.stdout: "cache_min_evict_age": 0, 2024-09-21T09:22:31.370 INFO:teuthology.orchestra.run.smithi078.stdout: "erasure_code_profile": "", 2024-09-21T09:22:31.370 INFO:teuthology.orchestra.run.smithi078.stdout: "hit_set_params": { 2024-09-21T09:22:31.370 INFO:teuthology.orchestra.run.smithi078.stdout: "type": "none" 2024-09-21T09:22:31.370 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T09:22:31.370 INFO:teuthology.orchestra.run.smithi078.stdout: "hit_set_period": 0, 2024-09-21T09:22:31.370 INFO:teuthology.orchestra.run.smithi078.stdout: "hit_set_count": 0, 2024-09-21T09:22:31.371 INFO:teuthology.orchestra.run.smithi078.stdout: "use_gmt_hitset": true, 2024-09-21T09:22:31.371 INFO:teuthology.orchestra.run.smithi078.stdout: "min_read_recency_for_promote": 0, 2024-09-21T09:22:31.371 INFO:teuthology.orchestra.run.smithi078.stdout: "min_write_recency_for_promote": 0, 2024-09-21T09:22:31.371 INFO:teuthology.orchestra.run.smithi078.stdout: "hit_set_grade_decay_rate": 0, 2024-09-21T09:22:31.371 INFO:teuthology.orchestra.run.smithi078.stdout: "hit_set_search_last_n": 0, 2024-09-21T09:22:31.371 INFO:teuthology.orchestra.run.smithi078.stdout: "grade_table": [], 2024-09-21T09:22:31.371 INFO:teuthology.orchestra.run.smithi078.stdout: "stripe_width": 0, 2024-09-21T09:22:31.371 INFO:teuthology.orchestra.run.smithi078.stdout: "expected_num_objects": 0, 2024-09-21T09:22:31.371 INFO:teuthology.orchestra.run.smithi078.stdout: "fast_read": false, 2024-09-21T09:22:31.371 INFO:teuthology.orchestra.run.smithi078.stdout: "options": { 2024-09-21T09:22:31.371 INFO:teuthology.orchestra.run.smithi078.stdout: "pg_num_max": 32, 2024-09-21T09:22:31.371 INFO:teuthology.orchestra.run.smithi078.stdout: "pg_num_min": 1 2024-09-21T09:22:31.371 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T09:22:31.372 INFO:teuthology.orchestra.run.smithi078.stdout: "application_metadata": { 2024-09-21T09:22:31.372 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": {} 2024-09-21T09:22:31.372 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T09:22:31.372 INFO:teuthology.orchestra.run.smithi078.stdout: "read_balance": { 2024-09-21T09:22:31.372 INFO:teuthology.orchestra.run.smithi078.stdout: "score_type": "Fair distribution", 2024-09-21T09:22:31.372 INFO:teuthology.orchestra.run.smithi078.stdout: "score_acting": 4, 2024-09-21T09:22:31.372 INFO:teuthology.orchestra.run.smithi078.stdout: "score_stable": 4, 2024-09-21T09:22:31.372 INFO:teuthology.orchestra.run.smithi078.stdout: "optimal_score": 0.5, 2024-09-21T09:22:31.372 INFO:teuthology.orchestra.run.smithi078.stdout: "raw_score_acting": 2, 2024-09-21T09:22:31.372 INFO:teuthology.orchestra.run.smithi078.stdout: "raw_score_stable": 2, 2024-09-21T09:22:31.372 INFO:teuthology.orchestra.run.smithi078.stdout: "primary_affinity_weighted": 1, 2024-09-21T09:22:31.372 INFO:teuthology.orchestra.run.smithi078.stdout: "average_primary_affinity": 1, 2024-09-21T09:22:31.372 INFO:teuthology.orchestra.run.smithi078.stdout: "average_primary_affinity_weighted": 1 2024-09-21T09:22:31.372 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-21T09:22:31.372 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T09:22:31.373 INFO:teuthology.orchestra.run.smithi078.stdout: { 2024-09-21T09:22:31.373 INFO:teuthology.orchestra.run.smithi078.stdout: "pool": 2, 2024-09-21T09:22:31.373 INFO:teuthology.orchestra.run.smithi078.stdout: "pool_name": "rbd", 2024-09-21T09:22:31.373 INFO:teuthology.orchestra.run.smithi078.stdout: "create_time": "2024-09-21T09:20:54.918007+0000", 2024-09-21T09:22:31.373 INFO:teuthology.orchestra.run.smithi078.stdout: "flags": 8193, 2024-09-21T09:22:31.373 INFO:teuthology.orchestra.run.smithi078.stdout: "flags_names": "hashpspool,selfmanaged_snaps", 2024-09-21T09:22:31.373 INFO:teuthology.orchestra.run.smithi078.stdout: "type": 1, 2024-09-21T09:22:31.373 INFO:teuthology.orchestra.run.smithi078.stdout: "size": 2, 2024-09-21T09:22:31.373 INFO:teuthology.orchestra.run.smithi078.stdout: "min_size": 1, 2024-09-21T09:22:31.373 INFO:teuthology.orchestra.run.smithi078.stdout: "crush_rule": 0, 2024-09-21T09:22:31.373 INFO:teuthology.orchestra.run.smithi078.stdout: "peering_crush_bucket_count": 0, 2024-09-21T09:22:31.373 INFO:teuthology.orchestra.run.smithi078.stdout: "peering_crush_bucket_target": 0, 2024-09-21T09:22:31.373 INFO:teuthology.orchestra.run.smithi078.stdout: "peering_crush_bucket_barrier": 0, 2024-09-21T09:22:31.374 INFO:teuthology.orchestra.run.smithi078.stdout: "peering_crush_bucket_mandatory_member": 2147483647, 2024-09-21T09:22:31.374 INFO:teuthology.orchestra.run.smithi078.stdout: "is_stretch_pool": false, 2024-09-21T09:22:31.374 INFO:teuthology.orchestra.run.smithi078.stdout: "object_hash": 2, 2024-09-21T09:22:31.374 INFO:teuthology.orchestra.run.smithi078.stdout: "pg_autoscale_mode": "off", 2024-09-21T09:22:31.374 INFO:teuthology.orchestra.run.smithi078.stdout: "pg_num": 8, 2024-09-21T09:22:31.374 INFO:teuthology.orchestra.run.smithi078.stdout: "pg_placement_num": 8, 2024-09-21T09:22:31.374 INFO:teuthology.orchestra.run.smithi078.stdout: "pg_placement_num_target": 8, 2024-09-21T09:22:31.374 INFO:teuthology.orchestra.run.smithi078.stdout: "pg_num_target": 8, 2024-09-21T09:22:31.374 INFO:teuthology.orchestra.run.smithi078.stdout: "pg_num_pending": 8, 2024-09-21T09:22:31.374 INFO:teuthology.orchestra.run.smithi078.stdout: "last_pg_merge_meta": { 2024-09-21T09:22:31.374 INFO:teuthology.orchestra.run.smithi078.stdout: "source_pgid": "0.0", 2024-09-21T09:22:31.374 INFO:teuthology.orchestra.run.smithi078.stdout: "ready_epoch": 0, 2024-09-21T09:22:31.374 INFO:teuthology.orchestra.run.smithi078.stdout: "last_epoch_started": 0, 2024-09-21T09:22:31.374 INFO:teuthology.orchestra.run.smithi078.stdout: "last_epoch_clean": 0, 2024-09-21T09:22:31.375 INFO:teuthology.orchestra.run.smithi078.stdout: "source_version": "0'0", 2024-09-21T09:22:31.375 INFO:teuthology.orchestra.run.smithi078.stdout: "target_version": "0'0" 2024-09-21T09:22:31.375 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T09:22:31.375 INFO:teuthology.orchestra.run.smithi078.stdout: "last_change": "17", 2024-09-21T09:22:31.375 INFO:teuthology.orchestra.run.smithi078.stdout: "last_force_op_resend": "0", 2024-09-21T09:22:31.375 INFO:teuthology.orchestra.run.smithi078.stdout: "last_force_op_resend_prenautilus": "0", 2024-09-21T09:22:31.375 INFO:teuthology.orchestra.run.smithi078.stdout: "last_force_op_resend_preluminous": "0", 2024-09-21T09:22:31.375 INFO:teuthology.orchestra.run.smithi078.stdout: "auid": 0, 2024-09-21T09:22:31.375 INFO:teuthology.orchestra.run.smithi078.stdout: "snap_mode": "selfmanaged", 2024-09-21T09:22:31.375 INFO:teuthology.orchestra.run.smithi078.stdout: "snap_seq": 3, 2024-09-21T09:22:31.375 INFO:teuthology.orchestra.run.smithi078.stdout: "snap_epoch": 17, 2024-09-21T09:22:31.375 INFO:teuthology.orchestra.run.smithi078.stdout: "pool_snaps": [], 2024-09-21T09:22:31.375 INFO:teuthology.orchestra.run.smithi078.stdout: "removed_snaps": "[]", 2024-09-21T09:22:31.375 INFO:teuthology.orchestra.run.smithi078.stdout: "quota_max_bytes": 0, 2024-09-21T09:22:31.376 INFO:teuthology.orchestra.run.smithi078.stdout: "quota_max_objects": 0, 2024-09-21T09:22:31.376 INFO:teuthology.orchestra.run.smithi078.stdout: "tiers": [], 2024-09-21T09:22:31.376 INFO:teuthology.orchestra.run.smithi078.stdout: "tier_of": -1, 2024-09-21T09:22:31.376 INFO:teuthology.orchestra.run.smithi078.stdout: "read_tier": -1, 2024-09-21T09:22:31.376 INFO:teuthology.orchestra.run.smithi078.stdout: "write_tier": -1, 2024-09-21T09:22:31.376 INFO:teuthology.orchestra.run.smithi078.stdout: "cache_mode": "none", 2024-09-21T09:22:31.376 INFO:teuthology.orchestra.run.smithi078.stdout: "target_max_bytes": 0, 2024-09-21T09:22:31.376 INFO:teuthology.orchestra.run.smithi078.stdout: "target_max_objects": 0, 2024-09-21T09:22:31.376 INFO:teuthology.orchestra.run.smithi078.stdout: "cache_target_dirty_ratio_micro": 400000, 2024-09-21T09:22:31.376 INFO:teuthology.orchestra.run.smithi078.stdout: "cache_target_dirty_high_ratio_micro": 600000, 2024-09-21T09:22:31.376 INFO:teuthology.orchestra.run.smithi078.stdout: "cache_target_full_ratio_micro": 800000, 2024-09-21T09:22:31.376 INFO:teuthology.orchestra.run.smithi078.stdout: "cache_min_flush_age": 0, 2024-09-21T09:22:31.376 INFO:teuthology.orchestra.run.smithi078.stdout: "cache_min_evict_age": 0, 2024-09-21T09:22:31.376 INFO:teuthology.orchestra.run.smithi078.stdout: "erasure_code_profile": "", 2024-09-21T09:22:31.376 INFO:teuthology.orchestra.run.smithi078.stdout: "hit_set_params": { 2024-09-21T09:22:31.377 INFO:teuthology.orchestra.run.smithi078.stdout: "type": "none" 2024-09-21T09:22:31.377 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T09:22:31.377 INFO:teuthology.orchestra.run.smithi078.stdout: "hit_set_period": 0, 2024-09-21T09:22:31.377 INFO:teuthology.orchestra.run.smithi078.stdout: "hit_set_count": 0, 2024-09-21T09:22:31.377 INFO:teuthology.orchestra.run.smithi078.stdout: "use_gmt_hitset": true, 2024-09-21T09:22:31.377 INFO:teuthology.orchestra.run.smithi078.stdout: "min_read_recency_for_promote": 0, 2024-09-21T09:22:31.377 INFO:teuthology.orchestra.run.smithi078.stdout: "min_write_recency_for_promote": 0, 2024-09-21T09:22:31.377 INFO:teuthology.orchestra.run.smithi078.stdout: "hit_set_grade_decay_rate": 0, 2024-09-21T09:22:31.377 INFO:teuthology.orchestra.run.smithi078.stdout: "hit_set_search_last_n": 0, 2024-09-21T09:22:31.377 INFO:teuthology.orchestra.run.smithi078.stdout: "grade_table": [], 2024-09-21T09:22:31.377 INFO:teuthology.orchestra.run.smithi078.stdout: "stripe_width": 0, 2024-09-21T09:22:31.377 INFO:teuthology.orchestra.run.smithi078.stdout: "expected_num_objects": 0, 2024-09-21T09:22:31.377 INFO:teuthology.orchestra.run.smithi078.stdout: "fast_read": false, 2024-09-21T09:22:31.377 INFO:teuthology.orchestra.run.smithi078.stdout: "options": {}, 2024-09-21T09:22:31.377 INFO:teuthology.orchestra.run.smithi078.stdout: "application_metadata": { 2024-09-21T09:22:31.377 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd": {} 2024-09-21T09:22:31.378 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T09:22:31.378 INFO:teuthology.orchestra.run.smithi078.stdout: "read_balance": { 2024-09-21T09:22:31.378 INFO:teuthology.orchestra.run.smithi078.stdout: "score_type": "Fair distribution", 2024-09-21T09:22:31.378 INFO:teuthology.orchestra.run.smithi078.stdout: "score_acting": 2.5, 2024-09-21T09:22:31.378 INFO:teuthology.orchestra.run.smithi078.stdout: "score_stable": 2.5, 2024-09-21T09:22:31.378 INFO:teuthology.orchestra.run.smithi078.stdout: "optimal_score": 1, 2024-09-21T09:22:31.378 INFO:teuthology.orchestra.run.smithi078.stdout: "raw_score_acting": 2.5, 2024-09-21T09:22:31.378 INFO:teuthology.orchestra.run.smithi078.stdout: "raw_score_stable": 2.5, 2024-09-21T09:22:31.378 INFO:teuthology.orchestra.run.smithi078.stdout: "primary_affinity_weighted": 1, 2024-09-21T09:22:31.378 INFO:teuthology.orchestra.run.smithi078.stdout: "average_primary_affinity": 1, 2024-09-21T09:22:31.378 INFO:teuthology.orchestra.run.smithi078.stdout: "average_primary_affinity_weighted": 1 2024-09-21T09:22:31.378 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-21T09:22:31.378 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-21T09:22:31.379 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-09-21T09:22:31.379 INFO:teuthology.orchestra.run.smithi078.stdout: "osds": [ 2024-09-21T09:22:31.379 INFO:teuthology.orchestra.run.smithi078.stdout: { 2024-09-21T09:22:31.379 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": 0, 2024-09-21T09:22:31.379 INFO:teuthology.orchestra.run.smithi078.stdout: "uuid": "6b18d222-0705-4495-a01c-3e2ba2ab4cff", 2024-09-21T09:22:31.379 INFO:teuthology.orchestra.run.smithi078.stdout: "up": 1, 2024-09-21T09:22:31.379 INFO:teuthology.orchestra.run.smithi078.stdout: "in": 1, 2024-09-21T09:22:31.379 INFO:teuthology.orchestra.run.smithi078.stdout: "weight": 1, 2024-09-21T09:22:31.379 INFO:teuthology.orchestra.run.smithi078.stdout: "primary_affinity": 1, 2024-09-21T09:22:31.379 INFO:teuthology.orchestra.run.smithi078.stdout: "last_clean_begin": 9, 2024-09-21T09:22:31.379 INFO:teuthology.orchestra.run.smithi078.stdout: "last_clean_end": 20, 2024-09-21T09:22:31.379 INFO:teuthology.orchestra.run.smithi078.stdout: "up_from": 26, 2024-09-21T09:22:31.379 INFO:teuthology.orchestra.run.smithi078.stdout: "up_thru": 26, 2024-09-21T09:22:31.379 INFO:teuthology.orchestra.run.smithi078.stdout: "down_at": 21, 2024-09-21T09:22:31.379 INFO:teuthology.orchestra.run.smithi078.stdout: "lost_at": 0, 2024-09-21T09:22:31.379 INFO:teuthology.orchestra.run.smithi078.stdout: "public_addrs": { 2024-09-21T09:22:31.380 INFO:teuthology.orchestra.run.smithi078.stdout: "addrvec": [ 2024-09-21T09:22:31.380 INFO:teuthology.orchestra.run.smithi078.stdout: { 2024-09-21T09:22:31.380 INFO:teuthology.orchestra.run.smithi078.stdout: "type": "v1", 2024-09-21T09:22:31.380 INFO:teuthology.orchestra.run.smithi078.stdout: "addr": "172.21.15.78:6800", 2024-09-21T09:22:31.380 INFO:teuthology.orchestra.run.smithi078.stdout: "nonce": 2538472627 2024-09-21T09:22:31.380 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-21T09:22:31.380 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2024-09-21T09:22:31.380 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T09:22:31.380 INFO:teuthology.orchestra.run.smithi078.stdout: "cluster_addrs": { 2024-09-21T09:22:31.380 INFO:teuthology.orchestra.run.smithi078.stdout: "addrvec": [ 2024-09-21T09:22:31.380 INFO:teuthology.orchestra.run.smithi078.stdout: { 2024-09-21T09:22:31.380 INFO:teuthology.orchestra.run.smithi078.stdout: "type": "v1", 2024-09-21T09:22:31.380 INFO:teuthology.orchestra.run.smithi078.stdout: "addr": "172.21.15.78:6801", 2024-09-21T09:22:31.380 INFO:teuthology.orchestra.run.smithi078.stdout: "nonce": 2538472627 2024-09-21T09:22:31.381 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-21T09:22:31.381 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2024-09-21T09:22:31.381 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T09:22:31.381 INFO:teuthology.orchestra.run.smithi078.stdout: "heartbeat_back_addrs": { 2024-09-21T09:22:31.381 INFO:teuthology.orchestra.run.smithi078.stdout: "addrvec": [ 2024-09-21T09:22:31.381 INFO:teuthology.orchestra.run.smithi078.stdout: { 2024-09-21T09:22:31.381 INFO:teuthology.orchestra.run.smithi078.stdout: "type": "v1", 2024-09-21T09:22:31.381 INFO:teuthology.orchestra.run.smithi078.stdout: "addr": "172.21.15.78:6805", 2024-09-21T09:22:31.381 INFO:teuthology.orchestra.run.smithi078.stdout: "nonce": 2538472627 2024-09-21T09:22:31.381 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-21T09:22:31.381 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2024-09-21T09:22:31.381 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T09:22:31.381 INFO:teuthology.orchestra.run.smithi078.stdout: "heartbeat_front_addrs": { 2024-09-21T09:22:31.381 INFO:teuthology.orchestra.run.smithi078.stdout: "addrvec": [ 2024-09-21T09:22:31.381 INFO:teuthology.orchestra.run.smithi078.stdout: { 2024-09-21T09:22:31.381 INFO:teuthology.orchestra.run.smithi078.stdout: "type": "v1", 2024-09-21T09:22:31.382 INFO:teuthology.orchestra.run.smithi078.stdout: "addr": "172.21.15.78:6803", 2024-09-21T09:22:31.382 INFO:teuthology.orchestra.run.smithi078.stdout: "nonce": 2538472627 2024-09-21T09:22:31.382 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-21T09:22:31.382 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2024-09-21T09:22:31.382 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T09:22:31.382 INFO:teuthology.orchestra.run.smithi078.stdout: "public_addr": "172.21.15.78:6800/2538472627", 2024-09-21T09:22:31.382 INFO:teuthology.orchestra.run.smithi078.stdout: "cluster_addr": "172.21.15.78:6801/2538472627", 2024-09-21T09:22:31.382 INFO:teuthology.orchestra.run.smithi078.stdout: "heartbeat_back_addr": "172.21.15.78:6805/2538472627", 2024-09-21T09:22:31.382 INFO:teuthology.orchestra.run.smithi078.stdout: "heartbeat_front_addr": "172.21.15.78:6803/2538472627", 2024-09-21T09:22:31.382 INFO:teuthology.orchestra.run.smithi078.stdout: "state": [ 2024-09-21T09:22:31.382 INFO:teuthology.orchestra.run.smithi078.stdout: "exists", 2024-09-21T09:22:31.382 INFO:teuthology.orchestra.run.smithi078.stdout: "up" 2024-09-21T09:22:31.382 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2024-09-21T09:22:31.382 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T09:22:31.383 INFO:teuthology.orchestra.run.smithi078.stdout: { 2024-09-21T09:22:31.383 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": 1, 2024-09-21T09:22:31.383 INFO:teuthology.orchestra.run.smithi078.stdout: "uuid": "2d15b0fa-66c2-4bda-b64b-d72cbbb16fb1", 2024-09-21T09:22:31.383 INFO:teuthology.orchestra.run.smithi078.stdout: "up": 1, 2024-09-21T09:22:31.383 INFO:teuthology.orchestra.run.smithi078.stdout: "in": 1, 2024-09-21T09:22:31.383 INFO:teuthology.orchestra.run.smithi078.stdout: "weight": 1, 2024-09-21T09:22:31.383 INFO:teuthology.orchestra.run.smithi078.stdout: "primary_affinity": 1, 2024-09-21T09:22:31.383 INFO:teuthology.orchestra.run.smithi078.stdout: "last_clean_begin": 9, 2024-09-21T09:22:31.383 INFO:teuthology.orchestra.run.smithi078.stdout: "last_clean_end": 20, 2024-09-21T09:22:31.383 INFO:teuthology.orchestra.run.smithi078.stdout: "up_from": 26, 2024-09-21T09:22:31.383 INFO:teuthology.orchestra.run.smithi078.stdout: "up_thru": 26, 2024-09-21T09:22:31.383 INFO:teuthology.orchestra.run.smithi078.stdout: "down_at": 22, 2024-09-21T09:22:31.383 INFO:teuthology.orchestra.run.smithi078.stdout: "lost_at": 0, 2024-09-21T09:22:31.384 INFO:teuthology.orchestra.run.smithi078.stdout: "public_addrs": { 2024-09-21T09:22:31.384 INFO:teuthology.orchestra.run.smithi078.stdout: "addrvec": [ 2024-09-21T09:22:31.384 INFO:teuthology.orchestra.run.smithi078.stdout: { 2024-09-21T09:22:31.385 INFO:teuthology.orchestra.run.smithi078.stdout: "type": "v1", 2024-09-21T09:22:31.385 INFO:teuthology.orchestra.run.smithi078.stdout: "addr": "172.21.15.78:6802", 2024-09-21T09:22:31.385 INFO:teuthology.orchestra.run.smithi078.stdout: "nonce": 3598780309 2024-09-21T09:22:31.385 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-21T09:22:31.385 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2024-09-21T09:22:31.385 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T09:22:31.386 INFO:teuthology.orchestra.run.smithi078.stdout: "cluster_addrs": { 2024-09-21T09:22:31.386 INFO:teuthology.orchestra.run.smithi078.stdout: "addrvec": [ 2024-09-21T09:22:31.386 INFO:teuthology.orchestra.run.smithi078.stdout: { 2024-09-21T09:22:31.386 INFO:teuthology.orchestra.run.smithi078.stdout: "type": "v1", 2024-09-21T09:22:31.386 INFO:teuthology.orchestra.run.smithi078.stdout: "addr": "172.21.15.78:6804", 2024-09-21T09:22:31.386 INFO:teuthology.orchestra.run.smithi078.stdout: "nonce": 3598780309 2024-09-21T09:22:31.386 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-21T09:22:31.386 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2024-09-21T09:22:31.386 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T09:22:31.386 INFO:teuthology.orchestra.run.smithi078.stdout: "heartbeat_back_addrs": { 2024-09-21T09:22:31.386 INFO:teuthology.orchestra.run.smithi078.stdout: "addrvec": [ 2024-09-21T09:22:31.386 INFO:teuthology.orchestra.run.smithi078.stdout: { 2024-09-21T09:22:31.386 INFO:teuthology.orchestra.run.smithi078.stdout: "type": "v1", 2024-09-21T09:22:31.386 INFO:teuthology.orchestra.run.smithi078.stdout: "addr": "172.21.15.78:6807", 2024-09-21T09:22:31.387 INFO:teuthology.orchestra.run.smithi078.stdout: "nonce": 3598780309 2024-09-21T09:22:31.387 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-21T09:22:31.387 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2024-09-21T09:22:31.387 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T09:22:31.387 INFO:teuthology.orchestra.run.smithi078.stdout: "heartbeat_front_addrs": { 2024-09-21T09:22:31.387 INFO:teuthology.orchestra.run.smithi078.stdout: "addrvec": [ 2024-09-21T09:22:31.387 INFO:teuthology.orchestra.run.smithi078.stdout: { 2024-09-21T09:22:31.387 INFO:teuthology.orchestra.run.smithi078.stdout: "type": "v1", 2024-09-21T09:22:31.387 INFO:teuthology.orchestra.run.smithi078.stdout: "addr": "172.21.15.78:6806", 2024-09-21T09:22:31.387 INFO:teuthology.orchestra.run.smithi078.stdout: "nonce": 3598780309 2024-09-21T09:22:31.387 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-21T09:22:31.387 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2024-09-21T09:22:31.387 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T09:22:31.387 INFO:teuthology.orchestra.run.smithi078.stdout: "public_addr": "172.21.15.78:6802/3598780309", 2024-09-21T09:22:31.388 INFO:teuthology.orchestra.run.smithi078.stdout: "cluster_addr": "172.21.15.78:6804/3598780309", 2024-09-21T09:22:31.388 INFO:teuthology.orchestra.run.smithi078.stdout: "heartbeat_back_addr": "172.21.15.78:6807/3598780309", 2024-09-21T09:22:31.388 INFO:teuthology.orchestra.run.smithi078.stdout: "heartbeat_front_addr": "172.21.15.78:6806/3598780309", 2024-09-21T09:22:31.388 INFO:teuthology.orchestra.run.smithi078.stdout: "state": [ 2024-09-21T09:22:31.388 INFO:teuthology.orchestra.run.smithi078.stdout: "exists", 2024-09-21T09:22:31.388 INFO:teuthology.orchestra.run.smithi078.stdout: "up" 2024-09-21T09:22:31.388 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2024-09-21T09:22:31.388 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T09:22:31.388 INFO:teuthology.orchestra.run.smithi078.stdout: { 2024-09-21T09:22:31.388 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": 2, 2024-09-21T09:22:31.388 INFO:teuthology.orchestra.run.smithi078.stdout: "uuid": "0bf75d8f-017d-488d-8a2e-c3fa3b2a29c2", 2024-09-21T09:22:31.388 INFO:teuthology.orchestra.run.smithi078.stdout: "up": 1, 2024-09-21T09:22:31.388 INFO:teuthology.orchestra.run.smithi078.stdout: "in": 1, 2024-09-21T09:22:31.388 INFO:teuthology.orchestra.run.smithi078.stdout: "weight": 1, 2024-09-21T09:22:31.389 INFO:teuthology.orchestra.run.smithi078.stdout: "primary_affinity": 1, 2024-09-21T09:22:31.389 INFO:teuthology.orchestra.run.smithi078.stdout: "last_clean_begin": 9, 2024-09-21T09:22:31.389 INFO:teuthology.orchestra.run.smithi078.stdout: "last_clean_end": 22, 2024-09-21T09:22:31.389 INFO:teuthology.orchestra.run.smithi078.stdout: "up_from": 27, 2024-09-21T09:22:31.389 INFO:teuthology.orchestra.run.smithi078.stdout: "up_thru": 27, 2024-09-21T09:22:31.389 INFO:teuthology.orchestra.run.smithi078.stdout: "down_at": 23, 2024-09-21T09:22:31.389 INFO:teuthology.orchestra.run.smithi078.stdout: "lost_at": 0, 2024-09-21T09:22:31.389 INFO:teuthology.orchestra.run.smithi078.stdout: "public_addrs": { 2024-09-21T09:22:31.389 INFO:teuthology.orchestra.run.smithi078.stdout: "addrvec": [ 2024-09-21T09:22:31.389 INFO:teuthology.orchestra.run.smithi078.stdout: { 2024-09-21T09:22:31.389 INFO:teuthology.orchestra.run.smithi078.stdout: "type": "v1", 2024-09-21T09:22:31.389 INFO:teuthology.orchestra.run.smithi078.stdout: "addr": "172.21.15.78:6808", 2024-09-21T09:22:31.389 INFO:teuthology.orchestra.run.smithi078.stdout: "nonce": 2733362473 2024-09-21T09:22:31.389 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-21T09:22:31.389 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2024-09-21T09:22:31.389 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T09:22:31.390 INFO:teuthology.orchestra.run.smithi078.stdout: "cluster_addrs": { 2024-09-21T09:22:31.390 INFO:teuthology.orchestra.run.smithi078.stdout: "addrvec": [ 2024-09-21T09:22:31.390 INFO:teuthology.orchestra.run.smithi078.stdout: { 2024-09-21T09:22:31.390 INFO:teuthology.orchestra.run.smithi078.stdout: "type": "v1", 2024-09-21T09:22:31.390 INFO:teuthology.orchestra.run.smithi078.stdout: "addr": "172.21.15.78:6809", 2024-09-21T09:22:31.390 INFO:teuthology.orchestra.run.smithi078.stdout: "nonce": 2733362473 2024-09-21T09:22:31.390 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-21T09:22:31.390 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2024-09-21T09:22:31.390 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T09:22:31.390 INFO:teuthology.orchestra.run.smithi078.stdout: "heartbeat_back_addrs": { 2024-09-21T09:22:31.390 INFO:teuthology.orchestra.run.smithi078.stdout: "addrvec": [ 2024-09-21T09:22:31.390 INFO:teuthology.orchestra.run.smithi078.stdout: { 2024-09-21T09:22:31.390 INFO:teuthology.orchestra.run.smithi078.stdout: "type": "v1", 2024-09-21T09:22:31.390 INFO:teuthology.orchestra.run.smithi078.stdout: "addr": "172.21.15.78:6811", 2024-09-21T09:22:31.391 INFO:teuthology.orchestra.run.smithi078.stdout: "nonce": 2733362473 2024-09-21T09:22:31.391 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-21T09:22:31.391 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2024-09-21T09:22:31.391 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T09:22:31.391 INFO:teuthology.orchestra.run.smithi078.stdout: "heartbeat_front_addrs": { 2024-09-21T09:22:31.391 INFO:teuthology.orchestra.run.smithi078.stdout: "addrvec": [ 2024-09-21T09:22:31.391 INFO:teuthology.orchestra.run.smithi078.stdout: { 2024-09-21T09:22:31.391 INFO:teuthology.orchestra.run.smithi078.stdout: "type": "v1", 2024-09-21T09:22:31.391 INFO:teuthology.orchestra.run.smithi078.stdout: "addr": "172.21.15.78:6810", 2024-09-21T09:22:31.391 INFO:teuthology.orchestra.run.smithi078.stdout: "nonce": 2733362473 2024-09-21T09:22:31.391 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-21T09:22:31.391 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2024-09-21T09:22:31.391 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T09:22:31.392 INFO:teuthology.orchestra.run.smithi078.stdout: "public_addr": "172.21.15.78:6808/2733362473", 2024-09-21T09:22:31.392 INFO:teuthology.orchestra.run.smithi078.stdout: "cluster_addr": "172.21.15.78:6809/2733362473", 2024-09-21T09:22:31.392 INFO:teuthology.orchestra.run.smithi078.stdout: "heartbeat_back_addr": "172.21.15.78:6811/2733362473", 2024-09-21T09:22:31.392 INFO:teuthology.orchestra.run.smithi078.stdout: "heartbeat_front_addr": "172.21.15.78:6810/2733362473", 2024-09-21T09:22:31.392 INFO:teuthology.orchestra.run.smithi078.stdout: "state": [ 2024-09-21T09:22:31.392 INFO:teuthology.orchestra.run.smithi078.stdout: "exists", 2024-09-21T09:22:31.392 INFO:teuthology.orchestra.run.smithi078.stdout: "up" 2024-09-21T09:22:31.392 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2024-09-21T09:22:31.392 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T09:22:31.392 INFO:teuthology.orchestra.run.smithi078.stdout: { 2024-09-21T09:22:31.392 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": 3, 2024-09-21T09:22:31.392 INFO:teuthology.orchestra.run.smithi078.stdout: "uuid": "5e3788c8-b89a-4614-b3ec-7ee1250ddbe0", 2024-09-21T09:22:31.392 INFO:teuthology.orchestra.run.smithi078.stdout: "up": 1, 2024-09-21T09:22:31.393 INFO:teuthology.orchestra.run.smithi078.stdout: "in": 1, 2024-09-21T09:22:31.393 INFO:teuthology.orchestra.run.smithi078.stdout: "weight": 1, 2024-09-21T09:22:31.393 INFO:teuthology.orchestra.run.smithi078.stdout: "primary_affinity": 1, 2024-09-21T09:22:31.393 INFO:teuthology.orchestra.run.smithi078.stdout: "last_clean_begin": 9, 2024-09-21T09:22:31.393 INFO:teuthology.orchestra.run.smithi078.stdout: "last_clean_end": 23, 2024-09-21T09:22:31.393 INFO:teuthology.orchestra.run.smithi078.stdout: "up_from": 28, 2024-09-21T09:22:31.393 INFO:teuthology.orchestra.run.smithi078.stdout: "up_thru": 23, 2024-09-21T09:22:31.393 INFO:teuthology.orchestra.run.smithi078.stdout: "down_at": 24, 2024-09-21T09:22:31.393 INFO:teuthology.orchestra.run.smithi078.stdout: "lost_at": 0, 2024-09-21T09:22:31.393 INFO:teuthology.orchestra.run.smithi078.stdout: "public_addrs": { 2024-09-21T09:22:31.393 INFO:teuthology.orchestra.run.smithi078.stdout: "addrvec": [ 2024-09-21T09:22:31.393 INFO:teuthology.orchestra.run.smithi078.stdout: { 2024-09-21T09:22:31.393 INFO:teuthology.orchestra.run.smithi078.stdout: "type": "v1", 2024-09-21T09:22:31.393 INFO:teuthology.orchestra.run.smithi078.stdout: "addr": "172.21.15.78:6812", 2024-09-21T09:22:31.393 INFO:teuthology.orchestra.run.smithi078.stdout: "nonce": 3499964365 2024-09-21T09:22:31.394 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-21T09:22:31.394 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2024-09-21T09:22:31.394 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T09:22:31.394 INFO:teuthology.orchestra.run.smithi078.stdout: "cluster_addrs": { 2024-09-21T09:22:31.394 INFO:teuthology.orchestra.run.smithi078.stdout: "addrvec": [ 2024-09-21T09:22:31.394 INFO:teuthology.orchestra.run.smithi078.stdout: { 2024-09-21T09:22:31.394 INFO:teuthology.orchestra.run.smithi078.stdout: "type": "v1", 2024-09-21T09:22:31.394 INFO:teuthology.orchestra.run.smithi078.stdout: "addr": "172.21.15.78:6813", 2024-09-21T09:22:31.394 INFO:teuthology.orchestra.run.smithi078.stdout: "nonce": 3499964365 2024-09-21T09:22:31.394 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-21T09:22:31.394 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2024-09-21T09:22:31.394 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T09:22:31.394 INFO:teuthology.orchestra.run.smithi078.stdout: "heartbeat_back_addrs": { 2024-09-21T09:22:31.395 INFO:teuthology.orchestra.run.smithi078.stdout: "addrvec": [ 2024-09-21T09:22:31.395 INFO:teuthology.orchestra.run.smithi078.stdout: { 2024-09-21T09:22:31.395 INFO:teuthology.orchestra.run.smithi078.stdout: "type": "v1", 2024-09-21T09:22:31.395 INFO:teuthology.orchestra.run.smithi078.stdout: "addr": "172.21.15.78:6815", 2024-09-21T09:22:31.395 INFO:teuthology.orchestra.run.smithi078.stdout: "nonce": 3499964365 2024-09-21T09:22:31.395 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-21T09:22:31.395 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2024-09-21T09:22:31.395 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T09:22:31.395 INFO:teuthology.orchestra.run.smithi078.stdout: "heartbeat_front_addrs": { 2024-09-21T09:22:31.395 INFO:teuthology.orchestra.run.smithi078.stdout: "addrvec": [ 2024-09-21T09:22:31.395 INFO:teuthology.orchestra.run.smithi078.stdout: { 2024-09-21T09:22:31.395 INFO:teuthology.orchestra.run.smithi078.stdout: "type": "v1", 2024-09-21T09:22:31.395 INFO:teuthology.orchestra.run.smithi078.stdout: "addr": "172.21.15.78:6814", 2024-09-21T09:22:31.395 INFO:teuthology.orchestra.run.smithi078.stdout: "nonce": 3499964365 2024-09-21T09:22:31.395 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-21T09:22:31.396 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2024-09-21T09:22:31.396 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T09:22:31.396 INFO:teuthology.orchestra.run.smithi078.stdout: "public_addr": "172.21.15.78:6812/3499964365", 2024-09-21T09:22:31.396 INFO:teuthology.orchestra.run.smithi078.stdout: "cluster_addr": "172.21.15.78:6813/3499964365", 2024-09-21T09:22:31.396 INFO:teuthology.orchestra.run.smithi078.stdout: "heartbeat_back_addr": "172.21.15.78:6815/3499964365", 2024-09-21T09:22:31.396 INFO:teuthology.orchestra.run.smithi078.stdout: "heartbeat_front_addr": "172.21.15.78:6814/3499964365", 2024-09-21T09:22:31.396 INFO:teuthology.orchestra.run.smithi078.stdout: "state": [ 2024-09-21T09:22:31.396 INFO:teuthology.orchestra.run.smithi078.stdout: "exists", 2024-09-21T09:22:31.396 INFO:teuthology.orchestra.run.smithi078.stdout: "up" 2024-09-21T09:22:31.396 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2024-09-21T09:22:31.396 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-21T09:22:31.396 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-09-21T09:22:31.396 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_xinfo": [ 2024-09-21T09:22:31.396 INFO:teuthology.orchestra.run.smithi078.stdout: { 2024-09-21T09:22:31.396 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": 0, 2024-09-21T09:22:31.396 INFO:teuthology.orchestra.run.smithi078.stdout: "down_stamp": "0.000000", 2024-09-21T09:22:31.397 INFO:teuthology.orchestra.run.smithi078.stdout: "laggy_probability": 0, 2024-09-21T09:22:31.397 INFO:teuthology.orchestra.run.smithi078.stdout: "laggy_interval": 0, 2024-09-21T09:22:31.397 INFO:teuthology.orchestra.run.smithi078.stdout: "features": 4540719139924082687, 2024-09-21T09:22:31.397 INFO:teuthology.orchestra.run.smithi078.stdout: "old_weight": 0, 2024-09-21T09:22:31.397 INFO:teuthology.orchestra.run.smithi078.stdout: "last_purged_snaps_scrub": "2024-09-21T09:20:45.499319+0000", 2024-09-21T09:22:31.397 INFO:teuthology.orchestra.run.smithi078.stdout: "dead_epoch": 21 2024-09-21T09:22:31.397 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T09:22:31.397 INFO:teuthology.orchestra.run.smithi078.stdout: { 2024-09-21T09:22:31.397 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": 1, 2024-09-21T09:22:31.397 INFO:teuthology.orchestra.run.smithi078.stdout: "down_stamp": "0.000000", 2024-09-21T09:22:31.397 INFO:teuthology.orchestra.run.smithi078.stdout: "laggy_probability": 0, 2024-09-21T09:22:31.397 INFO:teuthology.orchestra.run.smithi078.stdout: "laggy_interval": 0, 2024-09-21T09:22:31.397 INFO:teuthology.orchestra.run.smithi078.stdout: "features": 4540719139924082687, 2024-09-21T09:22:31.397 INFO:teuthology.orchestra.run.smithi078.stdout: "old_weight": 0, 2024-09-21T09:22:31.397 INFO:teuthology.orchestra.run.smithi078.stdout: "last_purged_snaps_scrub": "2024-09-21T09:20:45.489695+0000", 2024-09-21T09:22:31.397 INFO:teuthology.orchestra.run.smithi078.stdout: "dead_epoch": 22 2024-09-21T09:22:31.398 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T09:22:31.398 INFO:teuthology.orchestra.run.smithi078.stdout: { 2024-09-21T09:22:31.398 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": 2, 2024-09-21T09:22:31.398 INFO:teuthology.orchestra.run.smithi078.stdout: "down_stamp": "0.000000", 2024-09-21T09:22:31.398 INFO:teuthology.orchestra.run.smithi078.stdout: "laggy_probability": 0, 2024-09-21T09:22:31.398 INFO:teuthology.orchestra.run.smithi078.stdout: "laggy_interval": 0, 2024-09-21T09:22:31.398 INFO:teuthology.orchestra.run.smithi078.stdout: "features": 4540719139924082687, 2024-09-21T09:22:31.398 INFO:teuthology.orchestra.run.smithi078.stdout: "old_weight": 0, 2024-09-21T09:22:31.398 INFO:teuthology.orchestra.run.smithi078.stdout: "last_purged_snaps_scrub": "2024-09-21T09:20:45.541959+0000", 2024-09-21T09:22:31.398 INFO:teuthology.orchestra.run.smithi078.stdout: "dead_epoch": 23 2024-09-21T09:22:31.398 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T09:22:31.398 INFO:teuthology.orchestra.run.smithi078.stdout: { 2024-09-21T09:22:31.398 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": 3, 2024-09-21T09:22:31.398 INFO:teuthology.orchestra.run.smithi078.stdout: "down_stamp": "2024-09-21T09:22:23.923196+0000", 2024-09-21T09:22:31.398 INFO:teuthology.orchestra.run.smithi078.stdout: "laggy_probability": 0, 2024-09-21T09:22:31.399 INFO:teuthology.orchestra.run.smithi078.stdout: "laggy_interval": 0, 2024-09-21T09:22:31.399 INFO:teuthology.orchestra.run.smithi078.stdout: "features": 4540719139924082687, 2024-09-21T09:22:31.399 INFO:teuthology.orchestra.run.smithi078.stdout: "old_weight": 0, 2024-09-21T09:22:31.399 INFO:teuthology.orchestra.run.smithi078.stdout: "last_purged_snaps_scrub": "2024-09-21T09:20:45.476915+0000", 2024-09-21T09:22:31.399 INFO:teuthology.orchestra.run.smithi078.stdout: "dead_epoch": 0 2024-09-21T09:22:31.399 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-21T09:22:31.399 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-09-21T09:22:31.399 INFO:teuthology.orchestra.run.smithi078.stdout: "pg_upmap": [], 2024-09-21T09:22:31.399 INFO:teuthology.orchestra.run.smithi078.stdout: "pg_upmap_items": [], 2024-09-21T09:22:31.399 INFO:teuthology.orchestra.run.smithi078.stdout: "pg_upmap_primaries": [], 2024-09-21T09:22:31.399 INFO:teuthology.orchestra.run.smithi078.stdout: "pg_temp": [], 2024-09-21T09:22:31.399 INFO:teuthology.orchestra.run.smithi078.stdout: "primary_temp": [], 2024-09-21T09:22:31.399 INFO:teuthology.orchestra.run.smithi078.stdout: "blocklist": { 2024-09-21T09:22:31.399 INFO:teuthology.orchestra.run.smithi078.stdout: "172.21.15.78:6816/2602406893": "2024-09-22T09:22:16.387689+0000", 2024-09-21T09:22:31.400 INFO:teuthology.orchestra.run.smithi078.stdout: "172.21.15.78:0/1812672186": "2024-09-22T09:22:16.387689+0000", 2024-09-21T09:22:31.400 INFO:teuthology.orchestra.run.smithi078.stdout: "172.21.15.78:0/2894754949": "2024-09-22T09:22:16.387689+0000", 2024-09-21T09:22:31.400 INFO:teuthology.orchestra.run.smithi078.stdout: "172.21.15.78:0/2078474014": "2024-09-22T09:22:16.387689+0000", 2024-09-21T09:22:31.400 INFO:teuthology.orchestra.run.smithi078.stdout: "172.21.15.78:0/2453110290": "2024-09-22T09:22:16.387689+0000" 2024-09-21T09:22:31.400 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T09:22:31.400 INFO:teuthology.orchestra.run.smithi078.stdout: "range_blocklist": {}, 2024-09-21T09:22:31.400 INFO:teuthology.orchestra.run.smithi078.stdout: "erasure_code_profiles": { 2024-09-21T09:22:31.400 INFO:teuthology.orchestra.run.smithi078.stdout: "default": { 2024-09-21T09:22:31.400 INFO:teuthology.orchestra.run.smithi078.stdout: "crush-failure-domain": "osd", 2024-09-21T09:22:31.400 INFO:teuthology.orchestra.run.smithi078.stdout: "k": "2", 2024-09-21T09:22:31.400 INFO:teuthology.orchestra.run.smithi078.stdout: "m": "1", 2024-09-21T09:22:31.400 INFO:teuthology.orchestra.run.smithi078.stdout: "plugin": "jerasure", 2024-09-21T09:22:31.400 INFO:teuthology.orchestra.run.smithi078.stdout: "technique": "reed_sol_van" 2024-09-21T09:22:31.400 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-21T09:22:31.400 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T09:22:31.401 INFO:teuthology.orchestra.run.smithi078.stdout: "removed_snaps_queue": [], 2024-09-21T09:22:31.401 INFO:teuthology.orchestra.run.smithi078.stdout: "new_removed_snaps": [], 2024-09-21T09:22:31.401 INFO:teuthology.orchestra.run.smithi078.stdout: "new_purged_snaps": [], 2024-09-21T09:22:31.401 INFO:teuthology.orchestra.run.smithi078.stdout: "crush_node_flags": {}, 2024-09-21T09:22:31.401 INFO:teuthology.orchestra.run.smithi078.stdout: "device_class_flags": {}, 2024-09-21T09:22:31.401 INFO:teuthology.orchestra.run.smithi078.stdout: "stretch_mode": { 2024-09-21T09:22:31.401 INFO:teuthology.orchestra.run.smithi078.stdout: "stretch_mode_enabled": false, 2024-09-21T09:22:31.401 INFO:teuthology.orchestra.run.smithi078.stdout: "stretch_bucket_count": 0, 2024-09-21T09:22:31.401 INFO:teuthology.orchestra.run.smithi078.stdout: "degraded_stretch_mode": 0, 2024-09-21T09:22:31.401 INFO:teuthology.orchestra.run.smithi078.stdout: "recovering_stretch_mode": 0, 2024-09-21T09:22:31.401 INFO:teuthology.orchestra.run.smithi078.stdout: "stretch_mode_bucket": 0 2024-09-21T09:22:31.401 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-21T09:22:31.401 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-21T09:22:31.402 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.363+0000 7f136d17d640 1 -- 172.21.15.78:0/786850095 >> v1:172.21.15.78:6816/2186157269 conn(0x7f13540514b0 legacy=0x7f1354053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:31.402 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.363+0000 7f136d17d640 1 -- 172.21.15.78:0/786850095 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f136812cf60 msgr2=0x7f13680950e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:31.403 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.363+0000 7f136d17d640 1 --2- 172.21.15.78:0/786850095 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f136812cf60 0x7f13680950e0 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7f135c000c00 tx=0x7f135c0038a0 comp rx=0 tx=0).stop 2024-09-21T09:22:31.403 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.363+0000 7f136d17d640 1 -- 172.21.15.78:0/786850095 shutdown_connections 2024-09-21T09:22:31.403 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.363+0000 7f136d17d640 1 --2- 172.21.15.78:0/786850095 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f1368132200 0x7f1368095620 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:31.403 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.363+0000 7f136d17d640 1 --2- 172.21.15.78:0/786850095 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f136812cf60 0x7f13680950e0 unknown :-1 s=CLOSED pgs=42 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:31.403 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.363+0000 7f136d17d640 1 --2- 172.21.15.78:0/786850095 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f1368097500 0x7f1368094ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:31.403 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.363+0000 7f136d17d640 1 -- 172.21.15.78:0/786850095 >> 172.21.15.78:0/786850095 conn(0x7f136809b580 msgr2=0x7f136809c3b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:31.403 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.363+0000 7f136d17d640 1 -- 172.21.15.78:0/786850095 shutdown_connections 2024-09-21T09:22:31.403 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.363+0000 7f136d17d640 1 -- 172.21.15.78:0/786850095 wait complete. 2024-09-21T09:22:31.403 DEBUG:teuthology.orchestra.run.smithi078:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph fs dump' 2024-09-21T09:22:31.505 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.502+0000 7f8ec86ef640 1 -- 172.21.15.78:0/1610315058 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f8ec012c180 msgr2=0x7f8ec0091fc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:31.505 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.502+0000 7f8ec86ef640 1 --2- 172.21.15.78:0/1610315058 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f8ec012c180 0x7f8ec0091fc0 secure :-1 s=READY pgs=37 cs=0 l=1 rev1=1 crypto rx=0x7f8eb0009860 tx=0x7f8eb002a380 comp rx=0 tx=0).stop 2024-09-21T09:22:31.505 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.502+0000 7f8ec86ef640 1 -- 172.21.15.78:0/1610315058 shutdown_connections 2024-09-21T09:22:31.505 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.502+0000 7f8ec86ef640 1 --2- 172.21.15.78:0/1610315058 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f8ec012c180 0x7f8ec0091fc0 unknown :-1 s=CLOSED pgs=37 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:31.505 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.502+0000 7f8ec86ef640 1 --2- 172.21.15.78:0/1610315058 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f8ec0092500 0x7f8ec0092990 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:31.505 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.502+0000 7f8ec86ef640 1 --2- 172.21.15.78:0/1610315058 >> v2:172.21.15.78:3300/0 conn(0x7f8ec012b860 0x7f8ec012bc40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:31.505 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.502+0000 7f8ec86ef640 1 -- 172.21.15.78:0/1610315058 >> 172.21.15.78:0/1610315058 conn(0x7f8ec009b580 msgr2=0x7f8ec009b990 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:31.505 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.502+0000 7f8ec86ef640 1 -- 172.21.15.78:0/1610315058 shutdown_connections 2024-09-21T09:22:31.505 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.502+0000 7f8ec86ef640 1 -- 172.21.15.78:0/1610315058 wait complete. 2024-09-21T09:22:31.508 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.503+0000 7f8ec86ef640 1 Processor -- start 2024-09-21T09:22:31.508 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.503+0000 7f8ec86ef640 1 -- start start 2024-09-21T09:22:31.508 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.503+0000 7f8ec86ef640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f8ec0092500 0x7f8ec01308c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:31.508 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.503+0000 7f8ec86ef640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f8ec012b860 0x7f8ec0130e00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:31.508 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.503+0000 7f8ec86ef640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f8ec012c180 0x7f8ec0131340 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:31.508 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.503+0000 7f8ec86ef640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7f8ec0132840 con 0x7f8ec012b860 2024-09-21T09:22:31.508 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.503+0000 7f8ec86ef640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7f8ec0091ce0 con 0x7f8ec012c180 2024-09-21T09:22:31.508 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.503+0000 7f8ec86ef640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7f8ec01329c0 con 0x7f8ec0092500 2024-09-21T09:22:31.508 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.503+0000 7f8ec648b640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f8ec0092500 0x7f8ec01308c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:31.508 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.503+0000 7f8ec6c8c640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f8ec012c180 0x7f8ec0131340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:31.508 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.503+0000 7f8ec5c8a640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f8ec012b860 0x7f8ec0130e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:31.509 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.503+0000 7f8ec648b640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f8ec0092500 0x7f8ec01308c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3302/0 says I am v2:172.21.15.78:45434/0 (socket says 172.21.15.78:45434) 2024-09-21T09:22:31.509 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.503+0000 7f8ec648b640 1 -- 172.21.15.78:0/1766022885 learned_addr learned my addr 172.21.15.78:0/1766022885 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:31.509 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.504+0000 7f8ec648b640 1 -- 172.21.15.78:0/1766022885 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f8ec012c180 msgr2=0x7f8ec0131340 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:31.509 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.504+0000 7f8ec648b640 1 --2- 172.21.15.78:0/1766022885 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f8ec012c180 0x7f8ec0131340 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:31.509 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.504+0000 7f8ec648b640 1 -- 172.21.15.78:0/1766022885 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f8ec012b860 msgr2=0x7f8ec0130e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:31.509 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.504+0000 7f8ec648b640 1 --2- 172.21.15.78:0/1766022885 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f8ec012b860 0x7f8ec0130e00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:31.509 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.504+0000 7f8ec648b640 1 -- 172.21.15.78:0/1766022885 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f8ec006ef50 con 0x7f8ec0092500 2024-09-21T09:22:31.509 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.504+0000 7f8ec6c8c640 1 --2- 172.21.15.78:0/1766022885 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f8ec012c180 0x7f8ec0131340 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:22:31.509 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.504+0000 7f8ec648b640 1 --2- 172.21.15.78:0/1766022885 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f8ec0092500 0x7f8ec01308c0 secure :-1 s=READY pgs=38 cs=0 l=1 rev1=1 crypto rx=0x7f8ebc00afa0 tx=0x7f8ebc0095b0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:31.509 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.504+0000 7f8eb6ffd640 1 -- 172.21.15.78:0/1766022885 <== mon.2 v2:172.21.15.78:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8ebc003970 con 0x7f8ec0092500 2024-09-21T09:22:31.509 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.504+0000 7f8eb6ffd640 1 -- 172.21.15.78:0/1766022885 <== mon.2 v2:172.21.15.78:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f8ebc003b10 con 0x7f8ec0092500 2024-09-21T09:22:31.509 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.504+0000 7f8ec86ef640 1 -- 172.21.15.78:0/1766022885 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f8ec00747f0 con 0x7f8ec0092500 2024-09-21T09:22:31.509 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.504+0000 7f8eb6ffd640 1 -- 172.21.15.78:0/1766022885 <== mon.2 v2:172.21.15.78:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8ebc004020 con 0x7f8ec0092500 2024-09-21T09:22:31.509 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.504+0000 7f8ec86ef640 1 -- 172.21.15.78:0/1766022885 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f8ec0132260 con 0x7f8ec0092500 2024-09-21T09:22:31.510 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.505+0000 7f8ec86ef640 1 -- 172.21.15.78:0/1766022885 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f8ec0092990 con 0x7f8ec0092500 2024-09-21T09:22:31.510 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.505+0000 7f8eb6ffd640 1 -- 172.21.15.78:0/1766022885 <== mon.2 v2:172.21.15.78:3302/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f8ebc01c020 con 0x7f8ec0092500 2024-09-21T09:22:31.510 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.505+0000 7f8eb6ffd640 1 -- 172.21.15.78:0/1766022885 <== mon.2 v2:172.21.15.78:3302/0 5 ==== osd_map(29..29 src has 1..29) ==== 3027+0+0 (secure 0 0 0) 0x7f8ebc068c30 con 0x7f8ec0092500 2024-09-21T09:22:31.511 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.509+0000 7f8eb6ffd640 1 -- 172.21.15.78:0/1766022885 <== mon.2 v2:172.21.15.78:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f8ebc067ce0 con 0x7f8ec0092500 2024-09-21T09:22:31.683 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.681+0000 7f8ec86ef640 1 -- 172.21.15.78:0/1766022885 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_command({"prefix": "fs dump"} v 0) -- 0x7f8ec00749c0 con 0x7f8ec0092500 2024-09-21T09:22:31.684 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.681+0000 7f8eb6ffd640 1 -- 172.21.15.78:0/1766022885 <== mon.2 v2:172.21.15.78:3302/0 7 ==== mon_command_ack([{"prefix": "fs dump"}]=0 dumped fsmap epoch 1 v1) ==== 75+0+396 (secure 0 0 0) 0x7f8ebc067820 con 0x7f8ec0092500 2024-09-21T09:22:31.684 INFO:teuthology.orchestra.run.smithi078.stdout:e1 2024-09-21T09:22:31.684 INFO:teuthology.orchestra.run.smithi078.stdout:btime 1970-01-01T00:00:00:000000+0000 2024-09-21T09:22:31.684 INFO:teuthology.orchestra.run.smithi078.stdout:enable_multiple, ever_enabled_multiple: 1,1 2024-09-21T09:22:31.684 INFO:teuthology.orchestra.run.smithi078.stdout:default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-21T09:22:31.685 INFO:teuthology.orchestra.run.smithi078.stdout:legacy client fscid: -1 2024-09-21T09:22:31.685 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:22:31.685 INFO:teuthology.orchestra.run.smithi078.stdout:No filesystems configured 2024-09-21T09:22:31.685 INFO:teuthology.orchestra.run.smithi078.stderr:dumped fsmap epoch 1 2024-09-21T09:22:31.686 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.683+0000 7f8ec86ef640 1 -- 172.21.15.78:0/1766022885 >> v1:172.21.15.78:6816/2186157269 conn(0x7f8ea40514e0 legacy=0x7f8ea40539a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:31.686 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.683+0000 7f8ec86ef640 1 -- 172.21.15.78:0/1766022885 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f8ec0092500 msgr2=0x7f8ec01308c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:31.686 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.683+0000 7f8ec86ef640 1 --2- 172.21.15.78:0/1766022885 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f8ec0092500 0x7f8ec01308c0 secure :-1 s=READY pgs=38 cs=0 l=1 rev1=1 crypto rx=0x7f8ebc00afa0 tx=0x7f8ebc0095b0 comp rx=0 tx=0).stop 2024-09-21T09:22:31.686 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.683+0000 7f8ec86ef640 1 -- 172.21.15.78:0/1766022885 shutdown_connections 2024-09-21T09:22:31.686 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.683+0000 7f8ec86ef640 1 --2- 172.21.15.78:0/1766022885 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f8ec012c180 0x7f8ec0131340 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:31.686 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.683+0000 7f8ec86ef640 1 --2- 172.21.15.78:0/1766022885 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f8ec012b860 0x7f8ec0130e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:31.686 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.683+0000 7f8ec86ef640 1 --2- 172.21.15.78:0/1766022885 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f8ec0092500 0x7f8ec01308c0 unknown :-1 s=CLOSED pgs=38 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:31.686 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.683+0000 7f8ec86ef640 1 -- 172.21.15.78:0/1766022885 >> 172.21.15.78:0/1766022885 conn(0x7f8ec009b580 msgr2=0x7f8ec008ded0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:31.687 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.683+0000 7f8ec86ef640 1 -- 172.21.15.78:0/1766022885 shutdown_connections 2024-09-21T09:22:31.687 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.683+0000 7f8ec86ef640 1 -- 172.21.15.78:0/1766022885 wait complete. 2024-09-21T09:22:31.696 DEBUG:teuthology.orchestra.run.smithi078:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph osd require-osd-release squid' 2024-09-21T09:22:31.795 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.792+0000 7f983b00f640 1 -- 172.21.15.78:0/1848514821 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f98340928c0 msgr2=0x7f9834096e50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:31.795 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.792+0000 7f983b00f640 1 --2- 172.21.15.78:0/1848514821 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f98340928c0 0x7f9834096e50 secure :-1 s=READY pgs=95 cs=0 l=1 rev1=1 crypto rx=0x7f9820005fd0 tx=0x7f982002a4b0 comp rx=0 tx=0).stop 2024-09-21T09:22:31.796 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.792+0000 7f983b00f640 1 -- 172.21.15.78:0/1848514821 shutdown_connections 2024-09-21T09:22:31.796 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.792+0000 7f983b00f640 1 --2- 172.21.15.78:0/1848514821 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f98340928c0 0x7f9834096e50 unknown :-1 s=CLOSED pgs=95 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:31.796 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.792+0000 7f983b00f640 1 --2- 172.21.15.78:0/1848514821 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f9834097390 0x7f9834134a30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:31.796 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.792+0000 7f983b00f640 1 -- 172.21.15.78:0/1848514821 >> 172.21.15.78:0/1848514821 conn(0x7f983409f420 msgr2=0x7f983409f830 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:31.796 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.792+0000 7f983b00f640 1 -- 172.21.15.78:0/1848514821 shutdown_connections 2024-09-21T09:22:31.796 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.792+0000 7f983b00f640 1 -- 172.21.15.78:0/1848514821 wait complete. 2024-09-21T09:22:31.796 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.793+0000 7f983b00f640 1 Processor -- start 2024-09-21T09:22:31.796 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.793+0000 7f983b00f640 1 -- start start 2024-09-21T09:22:31.798 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.794+0000 7f983b00f640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f98340924e0 0x7f9834124ea0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:31.798 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.794+0000 7f983b00f640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f98340928c0 0x7f98341253e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:31.798 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.794+0000 7f983b00f640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f9834097390 0x7f9834125920 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:31.798 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.794+0000 7f983b00f640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7f98341356e0 con 0x7f98340928c0 2024-09-21T09:22:31.798 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.794+0000 7f983b00f640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7f9834135560 con 0x7f98340924e0 2024-09-21T09:22:31.798 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.794+0000 7f983b00f640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7f9834135860 con 0x7f9834097390 2024-09-21T09:22:31.799 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.794+0000 7f9833fff640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f98340928c0 0x7f98341253e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:31.799 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.794+0000 7f98395ac640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f9834097390 0x7f9834125920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:31.799 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.794+0000 7f9833fff640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f98340928c0 0x7f98341253e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3300/0 says I am v2:172.21.15.78:33932/0 (socket says 172.21.15.78:33932) 2024-09-21T09:22:31.799 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.794+0000 7f9833fff640 1 -- 172.21.15.78:0/1320191174 learned_addr learned my addr 172.21.15.78:0/1320191174 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:31.799 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.794+0000 7f9838dab640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f98340924e0 0x7f9834124ea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:31.799 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.794+0000 7f9833fff640 1 -- 172.21.15.78:0/1320191174 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f9834097390 msgr2=0x7f9834125920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:31.799 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.794+0000 7f9833fff640 1 --2- 172.21.15.78:0/1320191174 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f9834097390 0x7f9834125920 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:31.799 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.794+0000 7f9833fff640 1 -- 172.21.15.78:0/1320191174 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f98340924e0 msgr2=0x7f9834124ea0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:31.799 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.794+0000 7f9833fff640 1 --2- 172.21.15.78:0/1320191174 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f98340924e0 0x7f9834124ea0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:31.799 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.794+0000 7f9833fff640 1 -- 172.21.15.78:0/1320191174 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f9834126830 con 0x7f98340928c0 2024-09-21T09:22:31.799 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.794+0000 7f98395ac640 1 --2- 172.21.15.78:0/1320191174 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f9834097390 0x7f9834125920 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:22:31.799 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.794+0000 7f9838dab640 1 --2- 172.21.15.78:0/1320191174 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f98340924e0 0x7f9834124ea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:22:31.799 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.794+0000 7f9833fff640 1 --2- 172.21.15.78:0/1320191174 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f98340928c0 0x7f98341253e0 secure :-1 s=READY pgs=96 cs=0 l=1 rev1=1 crypto rx=0x7f9820006010 tx=0x7f98200038a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:31.799 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.795+0000 7f98317fa640 1 -- 172.21.15.78:0/1320191174 <== mon.0 v2:172.21.15.78:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f982003f070 con 0x7f98340928c0 2024-09-21T09:22:31.799 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.795+0000 7f98317fa640 1 -- 172.21.15.78:0/1320191174 <== mon.0 v2:172.21.15.78:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f98200047a0 con 0x7f98340928c0 2024-09-21T09:22:31.800 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.795+0000 7f98317fa640 1 -- 172.21.15.78:0/1320191174 <== mon.0 v2:172.21.15.78:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9820035030 con 0x7f98340928c0 2024-09-21T09:22:31.800 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.795+0000 7f983b00f640 1 -- 172.21.15.78:0/1320191174 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f9834127910 con 0x7f98340928c0 2024-09-21T09:22:31.800 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.795+0000 7f983b00f640 1 -- 172.21.15.78:0/1320191174 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f9834128960 con 0x7f98340928c0 2024-09-21T09:22:31.800 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.796+0000 7f983b00f640 1 -- 172.21.15.78:0/1320191174 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f98341284c0 con 0x7f98340928c0 2024-09-21T09:22:31.800 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.796+0000 7f98317fa640 1 -- 172.21.15.78:0/1320191174 <== mon.0 v2:172.21.15.78:3300/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f9820042070 con 0x7f98340928c0 2024-09-21T09:22:31.800 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.796+0000 7f98317fa640 1 -- 172.21.15.78:0/1320191174 <== mon.0 v2:172.21.15.78:3300/0 5 ==== osd_map(29..29 src has 1..29) ==== 3027+0+0 (secure 0 0 0) 0x7f9820089d70 con 0x7f98340928c0 2024-09-21T09:22:31.802 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.800+0000 7f98317fa640 1 -- 172.21.15.78:0/1320191174 <== mon.0 v2:172.21.15.78:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f9820088e20 con 0x7f98340928c0 2024-09-21T09:22:31.960 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:31.957+0000 7f983b00f640 1 -- 172.21.15.78:0/1320191174 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "osd require-osd-release", "release": "squid"} v 0) -- 0x7f9834096740 con 0x7f98340928c0 2024-09-21T09:22:32.477 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.475+0000 7f98317fa640 1 -- 172.21.15.78:0/1320191174 <== mon.0 v2:172.21.15.78:3300/0 7 ==== mon_command_ack([{"prefix": "osd require-osd-release", "release": "squid"}]=0 v30) ==== 91+0+0 (secure 0 0 0) 0x7f9820088960 con 0x7f98340928c0 2024-09-21T09:22:32.480 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.477+0000 7f983b00f640 1 -- 172.21.15.78:0/1320191174 >> v1:172.21.15.78:6816/2186157269 conn(0x7f98180514b0 legacy=0x7f9818053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:32.480 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.477+0000 7f983b00f640 1 -- 172.21.15.78:0/1320191174 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f98340928c0 msgr2=0x7f98341253e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:32.480 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.477+0000 7f983b00f640 1 --2- 172.21.15.78:0/1320191174 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f98340928c0 0x7f98341253e0 secure :-1 s=READY pgs=96 cs=0 l=1 rev1=1 crypto rx=0x7f9820006010 tx=0x7f98200038a0 comp rx=0 tx=0).stop 2024-09-21T09:22:32.480 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.477+0000 7f983b00f640 1 -- 172.21.15.78:0/1320191174 shutdown_connections 2024-09-21T09:22:32.480 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.477+0000 7f983b00f640 1 --2- 172.21.15.78:0/1320191174 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f9834097390 0x7f9834125920 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:32.480 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.477+0000 7f983b00f640 1 --2- 172.21.15.78:0/1320191174 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f98340928c0 0x7f98341253e0 unknown :-1 s=CLOSED pgs=96 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:32.480 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.477+0000 7f983b00f640 1 --2- 172.21.15.78:0/1320191174 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f98340924e0 0x7f9834124ea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:32.480 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.477+0000 7f983b00f640 1 -- 172.21.15.78:0/1320191174 >> 172.21.15.78:0/1320191174 conn(0x7f983409f420 msgr2=0x7f983409a1e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:32.480 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.477+0000 7f983b00f640 1 -- 172.21.15.78:0/1320191174 shutdown_connections 2024-09-21T09:22:32.480 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.477+0000 7f983b00f640 1 -- 172.21.15.78:0/1320191174 wait complete. 2024-09-21T09:22:32.492 DEBUG:teuthology.orchestra.run.smithi078:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'for f in `ceph osd pool ls` ; do ceph osd pool set $f pg_autoscale_mode off ; done' 2024-09-21T09:22:32.591 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.588+0000 7fb4db027640 1 -- 172.21.15.78:0/267979187 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fb4d4129b90 msgr2=0x7fb4d412a000 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:32.591 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.588+0000 7fb4db027640 1 --2- 172.21.15.78:0/267979187 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fb4d4129b90 0x7fb4d412a000 secure :-1 s=READY pgs=39 cs=0 l=1 rev1=1 crypto rx=0x7fb4cc009ca0 tx=0x7fb4cc02a910 comp rx=0 tx=0).stop 2024-09-21T09:22:32.591 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.588+0000 7fb4db027640 1 -- 172.21.15.78:0/267979187 shutdown_connections 2024-09-21T09:22:32.591 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.588+0000 7fb4db027640 1 --2- 172.21.15.78:0/267979187 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fb4d4129b90 0x7fb4d412a000 unknown :-1 s=CLOSED pgs=39 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:32.591 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.588+0000 7fb4db027640 1 --2- 172.21.15.78:0/267979187 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fb4d4096c70 0x7fb4d4129650 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:32.592 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.588+0000 7fb4db027640 1 --2- 172.21.15.78:0/267979187 >> v2:172.21.15.78:3300/0 conn(0x7fb4d4096350 0x7fb4d4096730 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:32.592 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.588+0000 7fb4db027640 1 -- 172.21.15.78:0/267979187 >> 172.21.15.78:0/267979187 conn(0x7fb4d409a6e0 msgr2=0x7fb4d4097b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:32.592 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.588+0000 7fb4db027640 1 -- 172.21.15.78:0/267979187 shutdown_connections 2024-09-21T09:22:32.592 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.588+0000 7fb4db027640 1 -- 172.21.15.78:0/267979187 wait complete. 2024-09-21T09:22:32.592 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.589+0000 7fb4db027640 1 Processor -- start 2024-09-21T09:22:32.592 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.589+0000 7fb4db027640 1 -- start start 2024-09-21T09:22:32.594 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.589+0000 7fb4db027640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fb4d4096350 0x7fb4d41c1800 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:32.594 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.589+0000 7fb4db027640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fb4d4096c70 0x7fb4d41c1d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:32.594 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.589+0000 7fb4db027640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fb4d4129b90 0x7fb4d41c2280 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:32.594 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.589+0000 7fb4db027640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7fb4d41333a0 con 0x7fb4d4096c70 2024-09-21T09:22:32.594 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.589+0000 7fb4db027640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7fb4d4133220 con 0x7fb4d4129b90 2024-09-21T09:22:32.594 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.589+0000 7fb4db027640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7fb4d4133520 con 0x7fb4d4096350 2024-09-21T09:22:32.594 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.590+0000 7fb4d8dc3640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fb4d4096350 0x7fb4d41c1800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:32.594 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.590+0000 7fb4d95c4640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fb4d4129b90 0x7fb4d41c2280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:32.594 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.590+0000 7fb4d8dc3640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fb4d4096350 0x7fb4d41c1800 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3302/0 says I am v2:172.21.15.78:35500/0 (socket says 172.21.15.78:35500) 2024-09-21T09:22:32.594 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.590+0000 7fb4d8dc3640 1 -- 172.21.15.78:0/2912467904 learned_addr learned my addr 172.21.15.78:0/2912467904 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:32.594 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.590+0000 7fb4cbfff640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fb4d4096c70 0x7fb4d41c1d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:32.595 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.590+0000 7fb4d8dc3640 1 -- 172.21.15.78:0/2912467904 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fb4d4129b90 msgr2=0x7fb4d41c2280 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:32.595 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.590+0000 7fb4d8dc3640 1 --2- 172.21.15.78:0/2912467904 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fb4d4129b90 0x7fb4d41c2280 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:32.595 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.590+0000 7fb4d8dc3640 1 -- 172.21.15.78:0/2912467904 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fb4d4096c70 msgr2=0x7fb4d41c1d40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:32.595 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.590+0000 7fb4d8dc3640 1 --2- 172.21.15.78:0/2912467904 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fb4d4096c70 0x7fb4d41c1d40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:32.595 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.590+0000 7fb4d8dc3640 1 -- 172.21.15.78:0/2912467904 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fb4d4074a30 con 0x7fb4d4096350 2024-09-21T09:22:32.595 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.590+0000 7fb4d8dc3640 1 --2- 172.21.15.78:0/2912467904 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fb4d4096350 0x7fb4d41c1800 secure :-1 s=READY pgs=40 cs=0 l=1 rev1=1 crypto rx=0x7fb4d000e9f0 tx=0x7fb4d000eeb0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:32.595 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.590+0000 7fb4c97fa640 1 -- 172.21.15.78:0/2912467904 <== mon.2 v2:172.21.15.78:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb4d000cd20 con 0x7fb4d4096350 2024-09-21T09:22:32.595 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.590+0000 7fb4c97fa640 1 -- 172.21.15.78:0/2912467904 <== mon.2 v2:172.21.15.78:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fb4d0010430 con 0x7fb4d4096350 2024-09-21T09:22:32.595 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.590+0000 7fb4c97fa640 1 -- 172.21.15.78:0/2912467904 <== mon.2 v2:172.21.15.78:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb4d0010840 con 0x7fb4d4096350 2024-09-21T09:22:32.595 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.590+0000 7fb4db027640 1 -- 172.21.15.78:0/2912467904 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fb4d41239c0 con 0x7fb4d4096350 2024-09-21T09:22:32.595 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.590+0000 7fb4db027640 1 -- 172.21.15.78:0/2912467904 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7fb4d4072c40 con 0x7fb4d4096350 2024-09-21T09:22:32.595 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.591+0000 7fb4c97fa640 1 -- 172.21.15.78:0/2912467904 <== mon.2 v2:172.21.15.78:3302/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7fb4d00109e0 con 0x7fb4d4096350 2024-09-21T09:22:32.595 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.591+0000 7fb4db027640 1 -- 172.21.15.78:0/2912467904 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fb4d4129650 con 0x7fb4d4096350 2024-09-21T09:22:32.596 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.592+0000 7fb4c97fa640 1 -- 172.21.15.78:0/2912467904 <== mon.2 v2:172.21.15.78:3302/0 5 ==== osd_map(30..30 src has 1..30) ==== 3027+0+0 (secure 0 0 0) 0x7fb4d0069bb0 con 0x7fb4d4096350 2024-09-21T09:22:32.597 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.595+0000 7fb4c97fa640 1 -- 172.21.15.78:0/2912467904 <== mon.2 v2:172.21.15.78:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7fb4d0068c60 con 0x7fb4d4096350 2024-09-21T09:22:32.750 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.748+0000 7fb4db027640 1 -- 172.21.15.78:0/2912467904 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_command({"prefix": "osd pool ls"} v 0) -- 0x7fb4d406a3e0 con 0x7fb4d4096350 2024-09-21T09:22:32.750 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.748+0000 7fb4c97fa640 1 -- 172.21.15.78:0/2912467904 <== mon.2 v2:172.21.15.78:3302/0 7 ==== mon_command_ack([{"prefix": "osd pool ls"}]=0 v30) ==== 59+0+9 (secure 0 0 0) 0x7fb4d00687a0 con 0x7fb4d4096350 2024-09-21T09:22:32.752 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.750+0000 7fb4db027640 1 -- 172.21.15.78:0/2912467904 >> v1:172.21.15.78:6816/2186157269 conn(0x7fb4c00514b0 legacy=0x7fb4c0053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:32.752 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.750+0000 7fb4db027640 1 -- 172.21.15.78:0/2912467904 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fb4d4096350 msgr2=0x7fb4d41c1800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:32.752 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.750+0000 7fb4db027640 1 --2- 172.21.15.78:0/2912467904 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fb4d4096350 0x7fb4d41c1800 secure :-1 s=READY pgs=40 cs=0 l=1 rev1=1 crypto rx=0x7fb4d000e9f0 tx=0x7fb4d000eeb0 comp rx=0 tx=0).stop 2024-09-21T09:22:32.753 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.750+0000 7fb4db027640 1 -- 172.21.15.78:0/2912467904 shutdown_connections 2024-09-21T09:22:32.753 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.750+0000 7fb4db027640 1 --2- 172.21.15.78:0/2912467904 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fb4d4129b90 0x7fb4d41c2280 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:32.753 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.750+0000 7fb4db027640 1 --2- 172.21.15.78:0/2912467904 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fb4d4096c70 0x7fb4d41c1d40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:32.753 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.750+0000 7fb4db027640 1 --2- 172.21.15.78:0/2912467904 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fb4d4096350 0x7fb4d41c1800 unknown :-1 s=CLOSED pgs=40 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:32.753 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.750+0000 7fb4db027640 1 -- 172.21.15.78:0/2912467904 >> 172.21.15.78:0/2912467904 conn(0x7fb4d409a6e0 msgr2=0x7fb4d408ded0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:32.753 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.750+0000 7fb4db027640 1 -- 172.21.15.78:0/2912467904 shutdown_connections 2024-09-21T09:22:32.753 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.750+0000 7fb4db027640 1 -- 172.21.15.78:0/2912467904 wait complete. 2024-09-21T09:22:32.840 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.837+0000 7f67dc427640 1 -- 172.21.15.78:0/3407357351 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f67d4095f50 msgr2=0x7f67d412eb60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:32.844 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.837+0000 7f67dc427640 1 --2- 172.21.15.78:0/3407357351 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f67d4095f50 0x7f67d412eb60 secure :-1 s=READY pgs=41 cs=0 l=1 rev1=1 crypto rx=0x7f67c4005fd0 tx=0x7f67c402a410 comp rx=0 tx=0).stop 2024-09-21T09:22:32.844 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.837+0000 7f67dc427640 1 -- 172.21.15.78:0/3407357351 shutdown_connections 2024-09-21T09:22:32.845 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.837+0000 7f67dc427640 1 --2- 172.21.15.78:0/3407357351 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f67d4095f50 0x7f67d412eb60 unknown :-1 s=CLOSED pgs=41 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:32.845 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.837+0000 7f67dc427640 1 --2- 172.21.15.78:0/3407357351 >> v2:172.21.15.78:3300/0 conn(0x7f67d412f0a0 0x7f67d412f550 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:32.845 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.837+0000 7f67dc427640 1 --2- 172.21.15.78:0/3407357351 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f67d4095630 0x7f67d4095a10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:32.845 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.837+0000 7f67dc427640 1 -- 172.21.15.78:0/3407357351 >> 172.21.15.78:0/3407357351 conn(0x7f67d40a15c0 msgr2=0x7f67d40a19d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:32.845 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.837+0000 7f67dc427640 1 -- 172.21.15.78:0/3407357351 shutdown_connections 2024-09-21T09:22:32.845 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.837+0000 7f67dc427640 1 -- 172.21.15.78:0/3407357351 wait complete. 2024-09-21T09:22:32.845 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.838+0000 7f67dc427640 1 Processor -- start 2024-09-21T09:22:32.845 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.838+0000 7f67dc427640 1 -- start start 2024-09-21T09:22:32.848 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.838+0000 7f67dc427640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f67d4095630 0x7f67d4073840 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:32.848 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.838+0000 7f67dc427640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f67d4095f50 0x7f67d4073d80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:32.848 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.838+0000 7f67dc427640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f67d412f0a0 0x7f67d40742c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:32.848 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.838+0000 7f67dc427640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7f67d4136780 con 0x7f67d412f0a0 2024-09-21T09:22:32.848 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.838+0000 7f67dc427640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7f67d4136600 con 0x7f67d4095f50 2024-09-21T09:22:32.848 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.838+0000 7f67dc427640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7f67d4136900 con 0x7f67d4095630 2024-09-21T09:22:32.848 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.838+0000 7f67da1c3640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f67d4095630 0x7f67d4073840 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:32.848 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.838+0000 7f67d99c2640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f67d4095f50 0x7f67d4073d80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:32.848 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.838+0000 7f67da1c3640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f67d4095630 0x7f67d4073840 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3302/0 says I am v2:172.21.15.78:35524/0 (socket says 172.21.15.78:35524) 2024-09-21T09:22:32.848 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.838+0000 7f67d99c2640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f67d4095f50 0x7f67d4073d80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3301/0 says I am v2:172.21.15.78:37296/0 (socket says 172.21.15.78:37296) 2024-09-21T09:22:32.849 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.838+0000 7f67da1c3640 1 -- 172.21.15.78:0/1651745556 learned_addr learned my addr 172.21.15.78:0/1651745556 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:32.849 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.838+0000 7f67da9c4640 1 --2- 172.21.15.78:0/1651745556 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f67d412f0a0 0x7f67d40742c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:32.849 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.838+0000 7f67da1c3640 1 -- 172.21.15.78:0/1651745556 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f67d4095f50 msgr2=0x7f67d4073d80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:32.849 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.838+0000 7f67da1c3640 1 --2- 172.21.15.78:0/1651745556 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f67d4095f50 0x7f67d4073d80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:32.849 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.838+0000 7f67da1c3640 1 -- 172.21.15.78:0/1651745556 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f67d412f0a0 msgr2=0x7f67d40742c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:32.849 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.838+0000 7f67da1c3640 1 --2- 172.21.15.78:0/1651745556 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f67d412f0a0 0x7f67d40742c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:32.849 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.838+0000 7f67da1c3640 1 -- 172.21.15.78:0/1651745556 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f67d40718f0 con 0x7f67d4095630 2024-09-21T09:22:32.849 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.838+0000 7f67d99c2640 1 --2- 172.21.15.78:0/1651745556 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f67d4095f50 0x7f67d4073d80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:22:32.849 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.838+0000 7f67da9c4640 1 --2- 172.21.15.78:0/1651745556 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f67d412f0a0 0x7f67d40742c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:22:32.849 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.839+0000 7f67da1c3640 1 --2- 172.21.15.78:0/1651745556 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f67d4095630 0x7f67d4073840 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7f67d000e790 tx=0x7f67d000ec50 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:32.849 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.839+0000 7f67caffd640 1 -- 172.21.15.78:0/1651745556 <== mon.2 v2:172.21.15.78:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f67d000eea0 con 0x7f67d4095630 2024-09-21T09:22:32.849 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.839+0000 7f67caffd640 1 -- 172.21.15.78:0/1651745556 <== mon.2 v2:172.21.15.78:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f67d000cdb0 con 0x7f67d4095630 2024-09-21T09:22:32.849 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.839+0000 7f67dc427640 1 -- 172.21.15.78:0/1651745556 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f67d4072cb0 con 0x7f67d4095630 2024-09-21T09:22:32.850 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.839+0000 7f67caffd640 1 -- 172.21.15.78:0/1651745556 <== mon.2 v2:172.21.15.78:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f67d0010690 con 0x7f67d4095630 2024-09-21T09:22:32.850 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.839+0000 7f67dc427640 1 -- 172.21.15.78:0/1651745556 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f67d41c4cc0 con 0x7f67d4095630 2024-09-21T09:22:32.850 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.840+0000 7f67dc427640 1 -- 172.21.15.78:0/1651745556 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f67d412e690 con 0x7f67d4095630 2024-09-21T09:22:32.850 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.840+0000 7f67caffd640 1 -- 172.21.15.78:0/1651745556 <== mon.2 v2:172.21.15.78:3302/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f67d0010830 con 0x7f67d4095630 2024-09-21T09:22:32.850 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.840+0000 7f67caffd640 1 -- 172.21.15.78:0/1651745556 <== mon.2 v2:172.21.15.78:3302/0 5 ==== osd_map(30..30 src has 1..30) ==== 3027+0+0 (secure 0 0 0) 0x7f67d00699e0 con 0x7f67d4095630 2024-09-21T09:22:32.850 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.844+0000 7f67caffd640 1 -- 172.21.15.78:0/1651745556 <== mon.2 v2:172.21.15.78:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f67d412e690 con 0x7f67d4095630 2024-09-21T09:22:32.994 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:32.992+0000 7f67dc427640 1 -- 172.21.15.78:0/1651745556 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_command({"prefix": "osd pool set", "pool": ".mgr", "var": "pg_autoscale_mode", "val": "off"} v 0) -- 0x7f67d412e1b0 con 0x7f67d4095630 2024-09-21T09:22:33.489 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:33.486+0000 7f67caffd640 1 -- 172.21.15.78:0/1651745556 <== mon.2 v2:172.21.15.78:3302/0 7 ==== mon_command_ack([{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_autoscale_mode", "val": "off"}]=0 set pool 1 pg_autoscale_mode to off v31) ==== 153+0+0 (secure 0 0 0) 0x7f67d412e1b0 con 0x7f67d4095630 2024-09-21T09:22:33.489 INFO:teuthology.orchestra.run.smithi078.stderr:set pool 1 pg_autoscale_mode to off 2024-09-21T09:22:33.491 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:33.488+0000 7f67dc427640 1 -- 172.21.15.78:0/1651745556 >> v1:172.21.15.78:6816/2186157269 conn(0x7f67b80514b0 legacy=0x7f67b8053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:33.491 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:33.488+0000 7f67dc427640 1 -- 172.21.15.78:0/1651745556 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f67d4095630 msgr2=0x7f67d4073840 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:33.491 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:33.488+0000 7f67dc427640 1 --2- 172.21.15.78:0/1651745556 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f67d4095630 0x7f67d4073840 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7f67d000e790 tx=0x7f67d000ec50 comp rx=0 tx=0).stop 2024-09-21T09:22:33.491 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:33.488+0000 7f67dc427640 1 -- 172.21.15.78:0/1651745556 shutdown_connections 2024-09-21T09:22:33.491 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:33.488+0000 7f67dc427640 1 --2- 172.21.15.78:0/1651745556 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f67d412f0a0 0x7f67d40742c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:33.491 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:33.488+0000 7f67dc427640 1 --2- 172.21.15.78:0/1651745556 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f67d4095f50 0x7f67d4073d80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:33.491 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:33.488+0000 7f67dc427640 1 --2- 172.21.15.78:0/1651745556 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f67d4095630 0x7f67d4073840 unknown :-1 s=CLOSED pgs=42 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:33.491 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:33.488+0000 7f67dc427640 1 -- 172.21.15.78:0/1651745556 >> 172.21.15.78:0/1651745556 conn(0x7f67d40a15c0 msgr2=0x7f67d409df20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:33.491 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:33.488+0000 7f67dc427640 1 -- 172.21.15.78:0/1651745556 shutdown_connections 2024-09-21T09:22:33.491 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:33.488+0000 7f67dc427640 1 -- 172.21.15.78:0/1651745556 wait complete. 2024-09-21T09:22:33.579 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:33.577+0000 7f99edc52640 1 -- 172.21.15.78:0/3730360554 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f99e8099060 msgr2=0x7f99e8099440 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:33.579 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:33.577+0000 7f99edc52640 1 --2- 172.21.15.78:0/3730360554 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f99e8099060 0x7f99e8099440 secure :-1 s=READY pgs=43 cs=0 l=1 rev1=1 crypto rx=0x7f99d8009be0 tx=0x7f99d802a800 comp rx=0 tx=0).stop 2024-09-21T09:22:33.580 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:33.577+0000 7f99edc52640 1 -- 172.21.15.78:0/3730360554 shutdown_connections 2024-09-21T09:22:33.580 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:33.577+0000 7f99edc52640 1 --2- 172.21.15.78:0/3730360554 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f99e8099da0 0x7f99e812f2e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:33.580 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:33.577+0000 7f99edc52640 1 --2- 172.21.15.78:0/3730360554 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f99e8099060 0x7f99e8099440 unknown :-1 s=CLOSED pgs=43 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:33.580 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:33.577+0000 7f99edc52640 1 -- 172.21.15.78:0/3730360554 >> 172.21.15.78:0/3730360554 conn(0x7f99e809f8b0 msgr2=0x7f99e809fcc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:33.580 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:33.577+0000 7f99edc52640 1 -- 172.21.15.78:0/3730360554 shutdown_connections 2024-09-21T09:22:33.580 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:33.577+0000 7f99edc52640 1 -- 172.21.15.78:0/3730360554 wait complete. 2024-09-21T09:22:33.582 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:33.578+0000 7f99edc52640 1 Processor -- start 2024-09-21T09:22:33.582 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:33.578+0000 7f99edc52640 1 -- start start 2024-09-21T09:22:33.582 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:33.578+0000 7f99edc52640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f99e8099060 0x7f99e80902d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:33.582 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:33.578+0000 7f99edc52640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f99e8099980 0x7f99e8090810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:33.582 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:33.578+0000 7f99edc52640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f99e8099da0 0x7f99e8090d50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:33.582 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:33.578+0000 7f99edc52640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7f99e81379d0 con 0x7f99e8099060 2024-09-21T09:22:33.582 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:33.578+0000 7f99edc52640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7f99e8137850 con 0x7f99e8099da0 2024-09-21T09:22:33.582 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:33.578+0000 7f99edc52640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7f99e8137b50 con 0x7f99e8099980 2024-09-21T09:22:33.582 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:33.578+0000 7f99e77fe640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f99e8099060 0x7f99e80902d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:33.583 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:33.578+0000 7f99e77fe640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f99e8099060 0x7f99e80902d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3300/0 says I am v2:172.21.15.78:44398/0 (socket says 172.21.15.78:44398) 2024-09-21T09:22:33.583 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:33.578+0000 7f99e77fe640 1 -- 172.21.15.78:0/4288855785 learned_addr learned my addr 172.21.15.78:0/4288855785 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:33.583 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:33.578+0000 7f99e7fff640 1 --2- 172.21.15.78:0/4288855785 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f99e8099da0 0x7f99e8090d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:33.583 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:33.578+0000 7f99e6ffd640 1 --2- 172.21.15.78:0/4288855785 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f99e8099980 0x7f99e8090810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:33.583 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:33.579+0000 7f99e77fe640 1 -- 172.21.15.78:0/4288855785 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f99e8099980 msgr2=0x7f99e8090810 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:33.583 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:33.579+0000 7f99e77fe640 1 --2- 172.21.15.78:0/4288855785 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f99e8099980 0x7f99e8090810 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:33.583 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:33.579+0000 7f99e77fe640 1 -- 172.21.15.78:0/4288855785 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f99e8099da0 msgr2=0x7f99e8090d50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:33.583 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:33.579+0000 7f99e77fe640 1 --2- 172.21.15.78:0/4288855785 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f99e8099da0 0x7f99e8090d50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:33.583 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:33.579+0000 7f99e77fe640 1 -- 172.21.15.78:0/4288855785 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f99e8093120 con 0x7f99e8099060 2024-09-21T09:22:33.583 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:33.579+0000 7f99e77fe640 1 --2- 172.21.15.78:0/4288855785 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f99e8099060 0x7f99e80902d0 secure :-1 s=READY pgs=98 cs=0 l=1 rev1=1 crypto rx=0x7f99d8035040 tx=0x7f99d8033e80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:33.583 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:33.579+0000 7f99cffff640 1 -- 172.21.15.78:0/4288855785 <== mon.0 v2:172.21.15.78:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f99d8042020 con 0x7f99e8099060 2024-09-21T09:22:33.583 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:33.579+0000 7f99cffff640 1 -- 172.21.15.78:0/4288855785 <== mon.0 v2:172.21.15.78:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f99d8004040 con 0x7f99e8099060 2024-09-21T09:22:33.583 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:33.579+0000 7f99edc52640 1 -- 172.21.15.78:0/4288855785 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f99e80951b0 con 0x7f99e8099060 2024-09-21T09:22:33.583 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:33.579+0000 7f99cffff640 1 -- 172.21.15.78:0/4288855785 <== mon.0 v2:172.21.15.78:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f99d8037040 con 0x7f99e8099060 2024-09-21T09:22:33.583 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:33.579+0000 7f99edc52640 1 -- 172.21.15.78:0/4288855785 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f99e8093820 con 0x7f99e8099060 2024-09-21T09:22:33.584 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:33.580+0000 7f99edc52640 1 -- 172.21.15.78:0/4288855785 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f99e80927c0 con 0x7f99e8099060 2024-09-21T09:22:33.584 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:33.580+0000 7f99cffff640 1 -- 172.21.15.78:0/4288855785 <== mon.0 v2:172.21.15.78:3300/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f99d8045020 con 0x7f99e8099060 2024-09-21T09:22:33.584 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:33.580+0000 7f99cffff640 1 -- 172.21.15.78:0/4288855785 <== mon.0 v2:172.21.15.78:3300/0 5 ==== osd_map(31..31 src has 1..31) ==== 3027+0+0 (secure 0 0 0) 0x7f99d8089dd0 con 0x7f99e8099060 2024-09-21T09:22:33.586 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:33.584+0000 7f99cffff640 1 -- 172.21.15.78:0/4288855785 <== mon.0 v2:172.21.15.78:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f99d804c030 con 0x7f99e8099060 2024-09-21T09:22:33.734 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:33.732+0000 7f99edc52640 1 -- 172.21.15.78:0/4288855785 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "osd pool set", "pool": "rbd", "var": "pg_autoscale_mode", "val": "off"} v 0) -- 0x7f99e8099440 con 0x7f99e8099060 2024-09-21T09:22:34.487 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.484+0000 7f99cffff640 1 -- 172.21.15.78:0/4288855785 <== mon.0 v2:172.21.15.78:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool set", "pool": "rbd", "var": "pg_autoscale_mode", "val": "off"}]=0 set pool 2 pg_autoscale_mode to off v32) ==== 152+0+0 (secure 0 0 0) 0x7f99d808b020 con 0x7f99e8099060 2024-09-21T09:22:34.487 INFO:teuthology.orchestra.run.smithi078.stderr:set pool 2 pg_autoscale_mode to off 2024-09-21T09:22:34.490 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.487+0000 7f99edc52640 1 -- 172.21.15.78:0/4288855785 >> v1:172.21.15.78:6816/2186157269 conn(0x7f99d40514b0 legacy=0x7f99d4053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:34.490 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.487+0000 7f99edc52640 1 -- 172.21.15.78:0/4288855785 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f99e8099060 msgr2=0x7f99e80902d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:34.490 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.487+0000 7f99edc52640 1 --2- 172.21.15.78:0/4288855785 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f99e8099060 0x7f99e80902d0 secure :-1 s=READY pgs=98 cs=0 l=1 rev1=1 crypto rx=0x7f99d8035040 tx=0x7f99d8033e80 comp rx=0 tx=0).stop 2024-09-21T09:22:34.490 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.487+0000 7f99edc52640 1 -- 172.21.15.78:0/4288855785 shutdown_connections 2024-09-21T09:22:34.490 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.487+0000 7f99edc52640 1 --2- 172.21.15.78:0/4288855785 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f99e8099da0 0x7f99e8090d50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:34.490 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.487+0000 7f99edc52640 1 --2- 172.21.15.78:0/4288855785 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f99e8099980 0x7f99e8090810 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:34.490 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.487+0000 7f99edc52640 1 --2- 172.21.15.78:0/4288855785 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f99e8099060 0x7f99e80902d0 unknown :-1 s=CLOSED pgs=98 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:34.490 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.487+0000 7f99edc52640 1 -- 172.21.15.78:0/4288855785 >> 172.21.15.78:0/4288855785 conn(0x7f99e809f8b0 msgr2=0x7f99e809d2c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:34.490 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.487+0000 7f99edc52640 1 -- 172.21.15.78:0/4288855785 shutdown_connections 2024-09-21T09:22:34.490 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.487+0000 7f99edc52640 1 -- 172.21.15.78:0/4288855785 wait complete. 2024-09-21T09:22:34.504 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-09-21T09:22:34.523 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-21T09:22:34.523 INFO:tasks.ceph.ceph_manager.ceph:waiting for mgr available 2024-09-21T09:22:34.523 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph mgr dump --format=json 2024-09-21T09:22:34.629 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.626+0000 7f9c93ba9640 1 -- 172.21.15.78:0/99799015 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f9c8c12af70 msgr2=0x7f9c8c12b350 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:34.629 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.626+0000 7f9c93ba9640 1 --2- 172.21.15.78:0/99799015 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f9c8c12af70 0x7f9c8c12b350 secure :-1 s=READY pgs=44 cs=0 l=1 rev1=1 crypto rx=0x7f9c88009900 tx=0x7f9c8802a4f0 comp rx=0 tx=0).stop 2024-09-21T09:22:34.629 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.626+0000 7f9c93ba9640 1 -- 172.21.15.78:0/99799015 shutdown_connections 2024-09-21T09:22:34.629 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.626+0000 7f9c93ba9640 1 --2- 172.21.15.78:0/99799015 >> v2:172.21.15.78:3300/0 conn(0x7f9c8c095120 0x7f9c8c132e20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:34.629 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.626+0000 7f9c93ba9640 1 --2- 172.21.15.78:0/99799015 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f9c8c12af70 0x7f9c8c12b350 unknown :-1 s=CLOSED pgs=44 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:34.629 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.626+0000 7f9c93ba9640 1 -- 172.21.15.78:0/99799015 >> 172.21.15.78:0/99799015 conn(0x7f9c8c09a2e0 msgr2=0x7f9c8c097ad0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:34.630 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.626+0000 7f9c93ba9640 1 -- 172.21.15.78:0/99799015 shutdown_connections 2024-09-21T09:22:34.630 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.626+0000 7f9c93ba9640 1 -- 172.21.15.78:0/99799015 wait complete. 2024-09-21T09:22:34.632 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.627+0000 7f9c93ba9640 1 Processor -- start 2024-09-21T09:22:34.632 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.627+0000 7f9c93ba9640 1 -- start start 2024-09-21T09:22:34.632 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.628+0000 7f9c93ba9640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f9c8c0945d0 0x7f9c8c090550 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:34.632 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.628+0000 7f9c93ba9640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f9c8c095120 0x7f9c8c090a90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:34.632 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.628+0000 7f9c93ba9640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f9c8c12af70 0x7f9c8c072a70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:34.632 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.628+0000 7f9c93ba9640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7f9c8c1336f0 con 0x7f9c8c095120 2024-09-21T09:22:34.632 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.628+0000 7f9c93ba9640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7f9c8c133570 con 0x7f9c8c0945d0 2024-09-21T09:22:34.632 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.628+0000 7f9c93ba9640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7f9c8c133870 con 0x7f9c8c12af70 2024-09-21T09:22:34.633 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.628+0000 7f9c91945640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f9c8c0945d0 0x7f9c8c090550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:34.633 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.628+0000 7f9c91144640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f9c8c095120 0x7f9c8c090a90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:34.633 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.628+0000 7f9c91945640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f9c8c0945d0 0x7f9c8c090550 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3301/0 says I am v2:172.21.15.78:37348/0 (socket says 172.21.15.78:37348) 2024-09-21T09:22:34.633 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.628+0000 7f9c91945640 1 -- 172.21.15.78:0/2274752802 learned_addr learned my addr 172.21.15.78:0/2274752802 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:34.633 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.628+0000 7f9c92146640 1 --2- 172.21.15.78:0/2274752802 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f9c8c12af70 0x7f9c8c072a70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:34.633 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.628+0000 7f9c91144640 1 -- 172.21.15.78:0/2274752802 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f9c8c12af70 msgr2=0x7f9c8c072a70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:34.633 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.628+0000 7f9c91144640 1 --2- 172.21.15.78:0/2274752802 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f9c8c12af70 0x7f9c8c072a70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:34.633 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.628+0000 7f9c91144640 1 -- 172.21.15.78:0/2274752802 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f9c8c0945d0 msgr2=0x7f9c8c090550 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:34.633 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.628+0000 7f9c91144640 1 --2- 172.21.15.78:0/2274752802 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f9c8c0945d0 0x7f9c8c090550 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:34.633 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.628+0000 7f9c91144640 1 -- 172.21.15.78:0/2274752802 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f9c8c12a230 con 0x7f9c8c095120 2024-09-21T09:22:34.633 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.628+0000 7f9c91945640 1 --2- 172.21.15.78:0/2274752802 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f9c8c0945d0 0x7f9c8c090550 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:22:34.633 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.628+0000 7f9c91144640 1 --2- 172.21.15.78:0/2274752802 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f9c8c095120 0x7f9c8c090a90 secure :-1 s=READY pgs=100 cs=0 l=1 rev1=1 crypto rx=0x7f9c7c00c430 tx=0x7f9c7c00c830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:34.634 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.628+0000 7f9c827fc640 1 -- 172.21.15.78:0/2274752802 <== mon.0 v2:172.21.15.78:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9c7c017070 con 0x7f9c8c095120 2024-09-21T09:22:34.634 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.628+0000 7f9c827fc640 1 -- 172.21.15.78:0/2274752802 <== mon.0 v2:172.21.15.78:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f9c7c0053c0 con 0x7f9c8c095120 2024-09-21T09:22:34.634 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.628+0000 7f9c93ba9640 1 -- 172.21.15.78:0/2274752802 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f9c8c129680 con 0x7f9c8c095120 2024-09-21T09:22:34.634 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.628+0000 7f9c827fc640 1 -- 172.21.15.78:0/2274752802 <== mon.0 v2:172.21.15.78:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9c7c003850 con 0x7f9c8c095120 2024-09-21T09:22:34.634 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.629+0000 7f9c93ba9640 1 -- 172.21.15.78:0/2274752802 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f9c8c073190 con 0x7f9c8c095120 2024-09-21T09:22:34.634 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.630+0000 7f9c827fc640 1 -- 172.21.15.78:0/2274752802 <== mon.0 v2:172.21.15.78:3300/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f9c7c01a070 con 0x7f9c8c095120 2024-09-21T09:22:34.634 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.630+0000 7f9c93ba9640 1 -- 172.21.15.78:0/2274752802 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f9c8c12b350 con 0x7f9c8c095120 2024-09-21T09:22:34.634 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.630+0000 7f9c827fc640 1 -- 172.21.15.78:0/2274752802 <== mon.0 v2:172.21.15.78:3300/0 5 ==== osd_map(32..32 src has 1..32) ==== 3027+0+0 (secure 0 0 0) 0x7f9c7c068400 con 0x7f9c8c095120 2024-09-21T09:22:34.635 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.633+0000 7f9c827fc640 1 -- 172.21.15.78:0/2274752802 <== mon.0 v2:172.21.15.78:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f9c8c0967f0 con 0x7f9c8c095120 2024-09-21T09:22:34.815 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.812+0000 7f9c93ba9640 1 -- 172.21.15.78:0/2274752802 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "mgr dump", "format": "json"} v 0) -- 0x7f9c8c0699c0 con 0x7f9c8c095120 2024-09-21T09:22:34.816 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.814+0000 7f9c827fc640 1 -- 172.21.15.78:0/2274752802 <== mon.0 v2:172.21.15.78:3300/0 7 ==== mon_command_ack([{"prefix": "mgr dump", "format": "json"}]=0 v10) ==== 74+0+124221 (secure 0 0 0) 0x7f9c7c03f550 con 0x7f9c8c095120 2024-09-21T09:22:34.818 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:22:34.819 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.816+0000 7f9c93ba9640 1 -- 172.21.15.78:0/2274752802 >> v1:172.21.15.78:6816/2186157269 conn(0x7f9c700514b0 legacy=0x7f9c70053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:34.819 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.816+0000 7f9c93ba9640 1 -- 172.21.15.78:0/2274752802 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f9c8c095120 msgr2=0x7f9c8c090a90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:34.819 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.816+0000 7f9c93ba9640 1 --2- 172.21.15.78:0/2274752802 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f9c8c095120 0x7f9c8c090a90 secure :-1 s=READY pgs=100 cs=0 l=1 rev1=1 crypto rx=0x7f9c7c00c430 tx=0x7f9c7c00c830 comp rx=0 tx=0).stop 2024-09-21T09:22:34.819 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.816+0000 7f9c93ba9640 1 -- 172.21.15.78:0/2274752802 shutdown_connections 2024-09-21T09:22:34.820 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.816+0000 7f9c93ba9640 1 --2- 172.21.15.78:0/2274752802 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f9c8c12af70 0x7f9c8c072a70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:34.820 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.816+0000 7f9c93ba9640 1 --2- 172.21.15.78:0/2274752802 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f9c8c095120 0x7f9c8c090a90 unknown :-1 s=CLOSED pgs=100 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:34.820 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.816+0000 7f9c93ba9640 1 --2- 172.21.15.78:0/2274752802 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f9c8c0945d0 0x7f9c8c090550 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:34.820 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.817+0000 7f9c93ba9640 1 -- 172.21.15.78:0/2274752802 >> 172.21.15.78:0/2274752802 conn(0x7f9c8c09a2e0 msgr2=0x7f9c8c122190 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:34.820 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.817+0000 7f9c93ba9640 1 -- 172.21.15.78:0/2274752802 shutdown_connections 2024-09-21T09:22:34.820 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.817+0000 7f9c93ba9640 1 -- 172.21.15.78:0/2274752802 wait complete. 2024-09-21T09:22:34.829 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":10,"flags":0,"active_gid":14103,"active_name":"x","active_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6816","nonce":2186157269}]},"active_addr":"172.21.15.78:6816/2186157269","active_change":"2024-09-21T09:22:16.387807+0000","active_mgr_features":4540719139924082687,"available":true,"standbys":[{"gid":14112,"name":"y","mgr_features":4540719139924082687,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to, use commas to separate multiple","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_requests":{"name":"max_requests","type":"int","level":"advanced","flags":0,"default_value":"500","min":"","max":"","enum_allowed":[],"desc":"Maximum number of requests to keep in memory. When new request comes in, the oldest request will be removed if the number of requests exceeds the max request number.if un-finished request is removed, error message will be logged in the ceph-mgr log.","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"smb","can_run":true,"error_string":"","module_options":{"internal_store_backend":{"name":"internal_store_backend","type":"str","level":"dev","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"set internal store backend. for develoment and testing only","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_orchestration":{"name":"update_orchestration","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically update orchestration when smb resources are changed","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["iostat","nfs","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to, use commas to separate multiple","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_requests":{"name":"max_requests","type":"int","level":"advanced","flags":0,"default_value":"500","min":"","max":"","enum_allowed":[],"desc":"Maximum number of requests to keep in memory. When new request comes in, the oldest request will be removed if the number of requests exceeds the max request number.if un-finished request is removed, error message will be logged in the ceph-mgr log.","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"smb","can_run":true,"error_string":"","module_options":{"internal_store_backend":{"name":"internal_store_backend","type":"str","level":"dev","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"set internal store backend. for develoment and testing only","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_orchestration":{"name":"update_orchestration","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically update orchestration when smb resources are changed","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":20,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.78:0","nonce":1972265341}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.78:0","nonce":1418746293}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.78:0","nonce":2981910238}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.78:0","nonce":1629950143}]}]} 2024-09-21T09:22:34.832 INFO:tasks.ceph.ceph_manager.ceph:mgr available! 2024-09-21T09:22:34.832 INFO:tasks.ceph.ceph_manager.ceph:waiting for all up 2024-09-21T09:22:34.832 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-09-21T09:22:34.932 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.929+0000 7ff462c45640 1 -- 172.21.15.78:0/3246439607 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7ff45c0991f0 msgr2=0x7ff45c12bc20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:34.932 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.929+0000 7ff462c45640 1 --2- 172.21.15.78:0/3246439607 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7ff45c0991f0 0x7ff45c12bc20 secure :-1 s=READY pgs=102 cs=0 l=1 rev1=1 crypto rx=0x7ff450005fd0 tx=0x7ff45002a4b0 comp rx=0 tx=0).stop 2024-09-21T09:22:34.932 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.929+0000 7ff462c45640 1 -- 172.21.15.78:0/3246439607 shutdown_connections 2024-09-21T09:22:34.932 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.929+0000 7ff462c45640 1 --2- 172.21.15.78:0/3246439607 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7ff45c095df0 0x7ff45c133b10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:34.932 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.929+0000 7ff462c45640 1 --2- 172.21.15.78:0/3246439607 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7ff45c0991f0 0x7ff45c12bc20 unknown :-1 s=CLOSED pgs=102 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:34.932 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.929+0000 7ff462c45640 1 -- 172.21.15.78:0/3246439607 >> 172.21.15.78:0/3246439607 conn(0x7ff45c09a2e0 msgr2=0x7ff45c097ad0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:34.932 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.929+0000 7ff462c45640 1 -- 172.21.15.78:0/3246439607 shutdown_connections 2024-09-21T09:22:34.932 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.929+0000 7ff462c45640 1 -- 172.21.15.78:0/3246439607 wait complete. 2024-09-21T09:22:34.935 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.930+0000 7ff462c45640 1 Processor -- start 2024-09-21T09:22:34.935 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.930+0000 7ff462c45640 1 -- start start 2024-09-21T09:22:34.935 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.931+0000 7ff462c45640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7ff45c095df0 0x7ff45c072a90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:34.935 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.931+0000 7ff462c45640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7ff45c0991f0 0x7ff45c072fd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:34.935 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.931+0000 7ff462c45640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7ff45c12c160 0x7ff45c073510 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:34.935 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.931+0000 7ff462c45640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7ff45c1346e0 con 0x7ff45c095df0 2024-09-21T09:22:34.935 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.931+0000 7ff462c45640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7ff45c134560 con 0x7ff45c12c160 2024-09-21T09:22:34.935 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.931+0000 7ff462c45640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7ff45c134860 con 0x7ff45c0991f0 2024-09-21T09:22:34.935 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.931+0000 7ff4609e1640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7ff45c095df0 0x7ff45c072a90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:34.935 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.931+0000 7ff45bfff640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7ff45c0991f0 0x7ff45c072fd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:34.936 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.931+0000 7ff4609e1640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7ff45c095df0 0x7ff45c072a90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3300/0 says I am v2:172.21.15.78:44428/0 (socket says 172.21.15.78:44428) 2024-09-21T09:22:34.936 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.931+0000 7ff4609e1640 1 -- 172.21.15.78:0/1932460875 learned_addr learned my addr 172.21.15.78:0/1932460875 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:34.936 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.931+0000 7ff4611e2640 1 --2- 172.21.15.78:0/1932460875 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7ff45c12c160 0x7ff45c073510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:34.936 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.931+0000 7ff45bfff640 1 -- 172.21.15.78:0/1932460875 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7ff45c12c160 msgr2=0x7ff45c073510 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:34.936 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.931+0000 7ff45bfff640 1 --2- 172.21.15.78:0/1932460875 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7ff45c12c160 0x7ff45c073510 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:34.936 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.931+0000 7ff45bfff640 1 -- 172.21.15.78:0/1932460875 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7ff45c095df0 msgr2=0x7ff45c072a90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:34.936 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.931+0000 7ff45bfff640 1 --2- 172.21.15.78:0/1932460875 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7ff45c095df0 0x7ff45c072a90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:34.936 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.931+0000 7ff45bfff640 1 -- 172.21.15.78:0/1932460875 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ff45c133e60 con 0x7ff45c0991f0 2024-09-21T09:22:34.936 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.931+0000 7ff45bfff640 1 --2- 172.21.15.78:0/1932460875 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7ff45c0991f0 0x7ff45c072fd0 secure :-1 s=READY pgs=43 cs=0 l=1 rev1=1 crypto rx=0x7ff44800c430 tx=0x7ff44800c830 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:34.936 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.931+0000 7ff4597fa640 1 -- 172.21.15.78:0/1932460875 <== mon.2 v2:172.21.15.78:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff448017070 con 0x7ff45c0991f0 2024-09-21T09:22:34.936 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.931+0000 7ff4597fa640 1 -- 172.21.15.78:0/1932460875 <== mon.2 v2:172.21.15.78:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7ff4480053c0 con 0x7ff45c0991f0 2024-09-21T09:22:34.936 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.931+0000 7ff4597fa640 1 -- 172.21.15.78:0/1932460875 <== mon.2 v2:172.21.15.78:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff448003850 con 0x7ff45c0991f0 2024-09-21T09:22:34.936 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.931+0000 7ff462c45640 1 -- 172.21.15.78:0/1932460875 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7ff45c122920 con 0x7ff45c0991f0 2024-09-21T09:22:34.937 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.931+0000 7ff462c45640 1 -- 172.21.15.78:0/1932460875 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7ff45c0945d0 con 0x7ff45c0991f0 2024-09-21T09:22:34.937 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.932+0000 7ff462c45640 1 -- 172.21.15.78:0/1932460875 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7ff45c12bc20 con 0x7ff45c0991f0 2024-09-21T09:22:34.937 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.932+0000 7ff4597fa640 1 -- 172.21.15.78:0/1932460875 <== mon.2 v2:172.21.15.78:3302/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7ff44801a070 con 0x7ff45c0991f0 2024-09-21T09:22:34.937 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.933+0000 7ff4597fa640 1 -- 172.21.15.78:0/1932460875 <== mon.2 v2:172.21.15.78:3302/0 5 ==== osd_map(32..32 src has 1..32) ==== 3027+0+0 (secure 0 0 0) 0x7ff448068370 con 0x7ff45c0991f0 2024-09-21T09:22:34.938 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:34.936+0000 7ff4597fa640 1 -- 172.21.15.78:0/1932460875 <== mon.2 v2:172.21.15.78:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7ff448067420 con 0x7ff45c0991f0 2024-09-21T09:22:35.087 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.084+0000 7ff462c45640 1 -- 172.21.15.78:0/1932460875 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7ff45c0699c0 con 0x7ff45c0991f0 2024-09-21T09:22:35.087 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.085+0000 7ff4597fa640 1 -- 172.21.15.78:0/1932460875 <== mon.2 v2:172.21.15.78:3302/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v32) ==== 74+0+9175 (secure 0 0 0) 0x7ff448066f60 con 0x7ff45c0991f0 2024-09-21T09:22:35.087 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:22:35.087 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":32,"fsid":"3633ad52-1063-4406-97d9-d53c339f43ca","created":"2024-09-21T09:20:38.248415+0000","modified":"2024-09-21T09:22:34.481410+0000","last_up_change":"2024-09-21T09:22:30.459558+0000","last_in_change":"2024-09-21T09:20:40.330925+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":7,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":4,"require_min_compat_client":"reef","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T09:20:51.169034+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"31","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-21T09:20:54.918007+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"32","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":17,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"6b18d222-0705-4495-a01c-3e2ba2ab4cff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":9,"last_clean_end":20,"up_from":26,"up_thru":26,"down_at":21,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6800","nonce":2538472627}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6801","nonce":2538472627}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6805","nonce":2538472627}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6803","nonce":2538472627}]},"public_addr":"172.21.15.78:6800/2538472627","cluster_addr":"172.21.15.78:6801/2538472627","heartbeat_back_addr":"172.21.15.78:6805/2538472627","heartbeat_front_addr":"172.21.15.78:6803/2538472627","state":["exists","up"]},{"osd":1,"uuid":"2d15b0fa-66c2-4bda-b64b-d72cbbb16fb1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":9,"last_clean_end":20,"up_from":26,"up_thru":26,"down_at":22,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6802","nonce":3598780309}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6804","nonce":3598780309}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6807","nonce":3598780309}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6806","nonce":3598780309}]},"public_addr":"172.21.15.78:6802/3598780309","cluster_addr":"172.21.15.78:6804/3598780309","heartbeat_back_addr":"172.21.15.78:6807/3598780309","heartbeat_front_addr":"172.21.15.78:6806/3598780309","state":["exists","up"]},{"osd":2,"uuid":"0bf75d8f-017d-488d-8a2e-c3fa3b2a29c2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":9,"last_clean_end":22,"up_from":27,"up_thru":28,"down_at":23,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6808","nonce":2733362473}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6809","nonce":2733362473}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6811","nonce":2733362473}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6810","nonce":2733362473}]},"public_addr":"172.21.15.78:6808/2733362473","cluster_addr":"172.21.15.78:6809/2733362473","heartbeat_back_addr":"172.21.15.78:6811/2733362473","heartbeat_front_addr":"172.21.15.78:6810/2733362473","state":["exists","up"]},{"osd":3,"uuid":"5e3788c8-b89a-4614-b3ec-7ee1250ddbe0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":9,"last_clean_end":23,"up_from":28,"up_thru":28,"down_at":24,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6812","nonce":3499964365}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6813","nonce":3499964365}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6815","nonce":3499964365}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6814","nonce":3499964365}]},"public_addr":"172.21.15.78:6812/3499964365","cluster_addr":"172.21.15.78:6813/3499964365","heartbeat_back_addr":"172.21.15.78:6815/3499964365","heartbeat_front_addr":"172.21.15.78:6814/3499964365","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:20:45.499319+0000","dead_epoch":21},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:20:45.489695+0000","dead_epoch":22},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:20:45.541959+0000","dead_epoch":23},{"osd":3,"down_stamp":"2024-09-21T09:22:23.923196+0000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:20:45.476915+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.78:6816/2602406893":"2024-09-22T09:22:16.387689+0000","172.21.15.78:0/1812672186":"2024-09-22T09:22:16.387689+0000","172.21.15.78:0/2894754949":"2024-09-22T09:22:16.387689+0000","172.21.15.78:0/2078474014":"2024-09-22T09:22:16.387689+0000","172.21.15.78:0/2453110290":"2024-09-22T09:22:16.387689+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T09:22:35.089 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.087+0000 7ff462c45640 1 -- 172.21.15.78:0/1932460875 >> v1:172.21.15.78:6816/2186157269 conn(0x7ff4400514b0 legacy=0x7ff440053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:35.089 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.087+0000 7ff462c45640 1 -- 172.21.15.78:0/1932460875 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7ff45c0991f0 msgr2=0x7ff45c072fd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:35.089 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.087+0000 7ff462c45640 1 --2- 172.21.15.78:0/1932460875 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7ff45c0991f0 0x7ff45c072fd0 secure :-1 s=READY pgs=43 cs=0 l=1 rev1=1 crypto rx=0x7ff44800c430 tx=0x7ff44800c830 comp rx=0 tx=0).stop 2024-09-21T09:22:35.090 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.087+0000 7ff462c45640 1 -- 172.21.15.78:0/1932460875 shutdown_connections 2024-09-21T09:22:35.090 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.087+0000 7ff462c45640 1 --2- 172.21.15.78:0/1932460875 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7ff45c12c160 0x7ff45c073510 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:35.090 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.087+0000 7ff462c45640 1 --2- 172.21.15.78:0/1932460875 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7ff45c0991f0 0x7ff45c072fd0 unknown :-1 s=CLOSED pgs=43 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:35.090 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.087+0000 7ff462c45640 1 --2- 172.21.15.78:0/1932460875 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7ff45c095df0 0x7ff45c072a90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:35.090 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.087+0000 7ff462c45640 1 -- 172.21.15.78:0/1932460875 >> 172.21.15.78:0/1932460875 conn(0x7ff45c09a2e0 msgr2=0x7ff45c08bf80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:35.090 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.087+0000 7ff462c45640 1 -- 172.21.15.78:0/1932460875 shutdown_connections 2024-09-21T09:22:35.090 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.087+0000 7ff462c45640 1 -- 172.21.15.78:0/1932460875 wait complete. 2024-09-21T09:22:35.100 INFO:tasks.ceph.ceph_manager.ceph:all up! 2024-09-21T09:22:35.100 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-09-21T09:22:35.200 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.198+0000 7f61dcbd5640 1 -- 172.21.15.78:0/4186099612 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f61d81324f0 msgr2=0x7f61d81348e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:35.201 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.198+0000 7f61dcbd5640 1 --2- 172.21.15.78:0/4186099612 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f61d81324f0 0x7f61d81348e0 secure :-1 s=READY pgs=44 cs=0 l=1 rev1=1 crypto rx=0x7f61d0005fd0 tx=0x7f61d002a780 comp rx=0 tx=0).stop 2024-09-21T09:22:35.201 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.198+0000 7f61dcbd5640 1 -- 172.21.15.78:0/4186099612 shutdown_connections 2024-09-21T09:22:35.201 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.198+0000 7f61dcbd5640 1 --2- 172.21.15.78:0/4186099612 >> v2:172.21.15.78:3300/0 conn(0x7f61d812fbc0 0x7f61d8131fb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:35.201 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.198+0000 7f61dcbd5640 1 --2- 172.21.15.78:0/4186099612 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f61d81324f0 0x7f61d81348e0 unknown :-1 s=CLOSED pgs=44 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:35.201 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.198+0000 7f61dcbd5640 1 -- 172.21.15.78:0/4186099612 >> 172.21.15.78:0/4186099612 conn(0x7f61d809d480 msgr2=0x7f61d809d890 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:35.201 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.198+0000 7f61dcbd5640 1 -- 172.21.15.78:0/4186099612 shutdown_connections 2024-09-21T09:22:35.201 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.198+0000 7f61dcbd5640 1 -- 172.21.15.78:0/4186099612 wait complete. 2024-09-21T09:22:35.204 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.199+0000 7f61dcbd5640 1 Processor -- start 2024-09-21T09:22:35.204 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.199+0000 7f61dcbd5640 1 -- start start 2024-09-21T09:22:35.204 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.199+0000 7f61dcbd5640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f61d8129540 0x7f61d812b800 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:35.204 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.199+0000 7f61dcbd5640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f61d812fbc0 0x7f61d812bd40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:35.204 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.199+0000 7f61dcbd5640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f61d81324f0 0x7f61d812c280 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:35.204 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.199+0000 7f61dcbd5640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7f61d8135860 con 0x7f61d81324f0 2024-09-21T09:22:35.205 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.199+0000 7f61dcbd5640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7f61d81356e0 con 0x7f61d812fbc0 2024-09-21T09:22:35.205 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.199+0000 7f61dcbd5640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7f61d81359e0 con 0x7f61d8129540 2024-09-21T09:22:35.205 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.199+0000 7f61d659c640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f61d8129540 0x7f61d812b800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:35.205 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.199+0000 7f61cffff640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f61d812fbc0 0x7f61d812bd40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:35.205 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.199+0000 7f61d6d9d640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f61d81324f0 0x7f61d812c280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:35.205 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.199+0000 7f61d659c640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f61d8129540 0x7f61d812b800 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3302/0 says I am v2:172.21.15.78:35596/0 (socket says 172.21.15.78:35596) 2024-09-21T09:22:35.205 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.199+0000 7f61cffff640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f61d812fbc0 0x7f61d812bd40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3301/0 says I am v2:172.21.15.78:37370/0 (socket says 172.21.15.78:37370) 2024-09-21T09:22:35.205 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.199+0000 7f61d6d9d640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f61d81324f0 0x7f61d812c280 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3300/0 says I am v2:172.21.15.78:44440/0 (socket says 172.21.15.78:44440) 2024-09-21T09:22:35.205 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.199+0000 7f61d659c640 1 -- 172.21.15.78:0/3718762537 learned_addr learned my addr 172.21.15.78:0/3718762537 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:35.205 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.200+0000 7f61d659c640 1 -- 172.21.15.78:0/3718762537 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f61d812fbc0 msgr2=0x7f61d812bd40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:35.205 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.200+0000 7f61d659c640 1 --2- 172.21.15.78:0/3718762537 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f61d812fbc0 0x7f61d812bd40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:35.205 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.200+0000 7f61d659c640 1 -- 172.21.15.78:0/3718762537 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f61d81324f0 msgr2=0x7f61d812c280 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:35.205 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.200+0000 7f61d659c640 1 --2- 172.21.15.78:0/3718762537 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f61d81324f0 0x7f61d812c280 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:35.206 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.200+0000 7f61d659c640 1 -- 172.21.15.78:0/3718762537 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f61d812e090 con 0x7f61d8129540 2024-09-21T09:22:35.206 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.200+0000 7f61d6d9d640 1 --2- 172.21.15.78:0/3718762537 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f61d81324f0 0x7f61d812c280 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:22:35.206 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.200+0000 7f61d659c640 1 --2- 172.21.15.78:0/3718762537 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f61d8129540 0x7f61d812b800 secure :-1 s=READY pgs=45 cs=0 l=1 rev1=1 crypto rx=0x7f61c8001910 tx=0x7f61c8001dd0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:35.206 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.200+0000 7f61ceffd640 1 -- 172.21.15.78:0/3718762537 <== mon.2 v2:172.21.15.78:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f61c800e2e0 con 0x7f61d8129540 2024-09-21T09:22:35.206 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.200+0000 7f61ceffd640 1 -- 172.21.15.78:0/3718762537 <== mon.2 v2:172.21.15.78:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f61c8010630 con 0x7f61d8129540 2024-09-21T09:22:35.206 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.200+0000 7f61dcbd5640 1 -- 172.21.15.78:0/3718762537 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f61d812d890 con 0x7f61d8129540 2024-09-21T09:22:35.206 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.200+0000 7f61ceffd640 1 -- 172.21.15.78:0/3718762537 <== mon.2 v2:172.21.15.78:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f61c8010a80 con 0x7f61d8129540 2024-09-21T09:22:35.206 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.200+0000 7f61dcbd5640 1 -- 172.21.15.78:0/3718762537 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f61d806f3e0 con 0x7f61d8129540 2024-09-21T09:22:35.206 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.201+0000 7f61ceffd640 1 -- 172.21.15.78:0/3718762537 <== mon.2 v2:172.21.15.78:3302/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f61c8010c20 con 0x7f61d8129540 2024-09-21T09:22:35.206 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.201+0000 7f61dcbd5640 1 -- 172.21.15.78:0/3718762537 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f61d8128f00 con 0x7f61d8129540 2024-09-21T09:22:35.206 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.201+0000 7f61ceffd640 1 -- 172.21.15.78:0/3718762537 <== mon.2 v2:172.21.15.78:3302/0 5 ==== osd_map(32..32 src has 1..32) ==== 3027+0+0 (secure 0 0 0) 0x7f61c806cd70 con 0x7f61d8129540 2024-09-21T09:22:35.207 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.205+0000 7f61ceffd640 1 -- 172.21.15.78:0/3718762537 <== mon.2 v2:172.21.15.78:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f61d8128f00 con 0x7f61d8129540 2024-09-21T09:22:35.359 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.356+0000 7f61dcbd5640 1 -- 172.21.15.78:0/3718762537 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7f61d8129920 con 0x7f61d8129540 2024-09-21T09:22:35.359 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.357+0000 7f61ceffd640 1 -- 172.21.15.78:0/3718762537 <== mon.2 v2:172.21.15.78:3302/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v32) ==== 74+0+9175 (secure 0 0 0) 0x7f61d812e090 con 0x7f61d8129540 2024-09-21T09:22:35.359 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:22:35.359 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":32,"fsid":"3633ad52-1063-4406-97d9-d53c339f43ca","created":"2024-09-21T09:20:38.248415+0000","modified":"2024-09-21T09:22:34.481410+0000","last_up_change":"2024-09-21T09:22:30.459558+0000","last_in_change":"2024-09-21T09:20:40.330925+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":7,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":4,"require_min_compat_client":"reef","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T09:20:51.169034+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"31","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-21T09:20:54.918007+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"32","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":17,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"6b18d222-0705-4495-a01c-3e2ba2ab4cff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":9,"last_clean_end":20,"up_from":26,"up_thru":26,"down_at":21,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6800","nonce":2538472627}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6801","nonce":2538472627}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6805","nonce":2538472627}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6803","nonce":2538472627}]},"public_addr":"172.21.15.78:6800/2538472627","cluster_addr":"172.21.15.78:6801/2538472627","heartbeat_back_addr":"172.21.15.78:6805/2538472627","heartbeat_front_addr":"172.21.15.78:6803/2538472627","state":["exists","up"]},{"osd":1,"uuid":"2d15b0fa-66c2-4bda-b64b-d72cbbb16fb1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":9,"last_clean_end":20,"up_from":26,"up_thru":26,"down_at":22,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6802","nonce":3598780309}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6804","nonce":3598780309}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6807","nonce":3598780309}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6806","nonce":3598780309}]},"public_addr":"172.21.15.78:6802/3598780309","cluster_addr":"172.21.15.78:6804/3598780309","heartbeat_back_addr":"172.21.15.78:6807/3598780309","heartbeat_front_addr":"172.21.15.78:6806/3598780309","state":["exists","up"]},{"osd":2,"uuid":"0bf75d8f-017d-488d-8a2e-c3fa3b2a29c2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":9,"last_clean_end":22,"up_from":27,"up_thru":28,"down_at":23,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6808","nonce":2733362473}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6809","nonce":2733362473}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6811","nonce":2733362473}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6810","nonce":2733362473}]},"public_addr":"172.21.15.78:6808/2733362473","cluster_addr":"172.21.15.78:6809/2733362473","heartbeat_back_addr":"172.21.15.78:6811/2733362473","heartbeat_front_addr":"172.21.15.78:6810/2733362473","state":["exists","up"]},{"osd":3,"uuid":"5e3788c8-b89a-4614-b3ec-7ee1250ddbe0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":9,"last_clean_end":23,"up_from":28,"up_thru":28,"down_at":24,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6812","nonce":3499964365}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6813","nonce":3499964365}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6815","nonce":3499964365}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6814","nonce":3499964365}]},"public_addr":"172.21.15.78:6812/3499964365","cluster_addr":"172.21.15.78:6813/3499964365","heartbeat_back_addr":"172.21.15.78:6815/3499964365","heartbeat_front_addr":"172.21.15.78:6814/3499964365","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:20:45.499319+0000","dead_epoch":21},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:20:45.489695+0000","dead_epoch":22},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:20:45.541959+0000","dead_epoch":23},{"osd":3,"down_stamp":"2024-09-21T09:22:23.923196+0000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:20:45.476915+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.78:6816/2602406893":"2024-09-22T09:22:16.387689+0000","172.21.15.78:0/1812672186":"2024-09-22T09:22:16.387689+0000","172.21.15.78:0/2894754949":"2024-09-22T09:22:16.387689+0000","172.21.15.78:0/2078474014":"2024-09-22T09:22:16.387689+0000","172.21.15.78:0/2453110290":"2024-09-22T09:22:16.387689+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T09:22:35.361 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.359+0000 7f61dcbd5640 1 -- 172.21.15.78:0/3718762537 >> v1:172.21.15.78:6816/2186157269 conn(0x7f61b80514b0 legacy=0x7f61b8053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:35.362 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.359+0000 7f61dcbd5640 1 -- 172.21.15.78:0/3718762537 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f61d8129540 msgr2=0x7f61d812b800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:35.362 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.359+0000 7f61dcbd5640 1 --2- 172.21.15.78:0/3718762537 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f61d8129540 0x7f61d812b800 secure :-1 s=READY pgs=45 cs=0 l=1 rev1=1 crypto rx=0x7f61c8001910 tx=0x7f61c8001dd0 comp rx=0 tx=0).stop 2024-09-21T09:22:35.362 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.359+0000 7f61dcbd5640 1 -- 172.21.15.78:0/3718762537 shutdown_connections 2024-09-21T09:22:35.362 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.359+0000 7f61dcbd5640 1 --2- 172.21.15.78:0/3718762537 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f61d81324f0 0x7f61d812c280 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:35.362 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.359+0000 7f61dcbd5640 1 --2- 172.21.15.78:0/3718762537 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f61d812fbc0 0x7f61d812bd40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:35.362 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.359+0000 7f61dcbd5640 1 --2- 172.21.15.78:0/3718762537 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f61d8129540 0x7f61d812b800 unknown :-1 s=CLOSED pgs=45 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:35.362 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.359+0000 7f61dcbd5640 1 -- 172.21.15.78:0/3718762537 >> 172.21.15.78:0/3718762537 conn(0x7f61d809d480 msgr2=0x7f61d809a9f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:35.362 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.359+0000 7f61dcbd5640 1 -- 172.21.15.78:0/3718762537 shutdown_connections 2024-09-21T09:22:35.362 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.359+0000 7f61dcbd5640 1 -- 172.21.15.78:0/3718762537 wait complete. 2024-09-21T09:22:35.372 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.0 flush_pg_stats 2024-09-21T09:22:35.373 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.1 flush_pg_stats 2024-09-21T09:22:35.373 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.2 flush_pg_stats 2024-09-21T09:22:35.373 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.3 flush_pg_stats 2024-09-21T09:22:35.481 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.479+0000 7fb946e1d640 1 -- 172.21.15.78:0/2021634152 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fb940131240 msgr2=0x7fb940133630 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:35.481 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.479+0000 7fb946e1d640 1 --2- 172.21.15.78:0/2021634152 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fb940131240 0x7fb940133630 secure :-1 s=READY pgs=46 cs=0 l=1 rev1=1 crypto rx=0x7fb938009ca0 tx=0x7fb93802a8c0 comp rx=0 tx=0).stop 2024-09-21T09:22:35.482 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.479+0000 7fb946e1d640 1 -- 172.21.15.78:0/2021634152 shutdown_connections 2024-09-21T09:22:35.482 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.479+0000 7fb946e1d640 1 --2- 172.21.15.78:0/2021634152 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fb94012e910 0x7fb940130d00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:35.482 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.479+0000 7fb946e1d640 1 --2- 172.21.15.78:0/2021634152 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fb940131240 0x7fb940133630 unknown :-1 s=CLOSED pgs=46 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:35.482 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.479+0000 7fb946e1d640 1 -- 172.21.15.78:0/2021634152 >> 172.21.15.78:0/2021634152 conn(0x7fb94009c230 msgr2=0x7fb94009c640 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:35.482 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.479+0000 7fb946e1d640 1 -- 172.21.15.78:0/2021634152 shutdown_connections 2024-09-21T09:22:35.482 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.479+0000 7fb946e1d640 1 -- 172.21.15.78:0/2021634152 wait complete. 2024-09-21T09:22:35.483 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.479+0000 7f1efc33f640 1 -- 172.21.15.78:0/3760943918 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f1ef4096890 msgr2=0x7f1ef41271e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:35.483 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.479+0000 7f1efc33f640 1 --2- 172.21.15.78:0/3760943918 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f1ef4096890 0x7f1ef41271e0 secure :-1 s=READY pgs=107 cs=0 l=1 rev1=1 crypto rx=0x7f1ee4005fd0 tx=0x7f1ee402a540 comp rx=0 tx=0).stop 2024-09-21T09:22:35.483 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.479+0000 7f1efc33f640 1 -- 172.21.15.78:0/3760943918 shutdown_connections 2024-09-21T09:22:35.483 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.479+0000 7f1efc33f640 1 --2- 172.21.15.78:0/3760943918 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f1ef4096890 0x7f1ef41271e0 unknown :-1 s=CLOSED pgs=107 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:35.483 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.479+0000 7f1efc33f640 1 --2- 172.21.15.78:0/3760943918 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f1ef4095f70 0x7f1ef4096350 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:35.483 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.479+0000 7f1efc33f640 1 -- 172.21.15.78:0/3760943918 >> 172.21.15.78:0/3760943918 conn(0x7f1ef409afd0 msgr2=0x7f1ef409b3e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:35.483 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.479+0000 7f1efc33f640 1 -- 172.21.15.78:0/3760943918 shutdown_connections 2024-09-21T09:22:35.483 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.479+0000 7f1efc33f640 1 -- 172.21.15.78:0/3760943918 wait complete. 2024-09-21T09:22:35.484 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.479+0000 7ff36c351640 1 -- 172.21.15.78:0/3719655289 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7ff36412f1c0 msgr2=0x7ff36412f650 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:35.484 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.479+0000 7ff36c351640 1 --2- 172.21.15.78:0/3719655289 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7ff36412f1c0 0x7ff36412f650 secure :-1 s=READY pgs=45 cs=0 l=1 rev1=1 crypto rx=0x7ff35c009be0 tx=0x7ff35c02a760 comp rx=0 tx=0).stop 2024-09-21T09:22:35.484 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.479+0000 7fc82a3e7640 1 -- 172.21.15.78:0/215289945 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fc82406f7e0 msgr2=0x7fc82406fbc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:35.484 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.479+0000 7fc82a3e7640 1 --2- 172.21.15.78:0/215289945 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fc82406f7e0 0x7fc82406fbc0 secure :-1 s=READY pgs=108 cs=0 l=1 rev1=1 crypto rx=0x7fc8140098b0 tx=0x7fc81402a480 comp rx=0 tx=0).stop 2024-09-21T09:22:35.485 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.479+0000 7ff36c351640 1 -- 172.21.15.78:0/3719655289 shutdown_connections 2024-09-21T09:22:35.485 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.479+0000 7ff36c351640 1 --2- 172.21.15.78:0/3719655289 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7ff36412f1c0 0x7ff36412f650 unknown :-1 s=CLOSED pgs=45 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:35.485 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.479+0000 7ff36c351640 1 --2- 172.21.15.78:0/3719655289 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7ff364098d40 0x7ff364099120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:35.485 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.479+0000 7ff36c351640 1 -- 172.21.15.78:0/3719655289 >> 172.21.15.78:0/3719655289 conn(0x7ff3640a1790 msgr2=0x7ff3640a1ba0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:35.485 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.479+0000 7ff36c351640 1 -- 172.21.15.78:0/3719655289 shutdown_connections 2024-09-21T09:22:35.486 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.480+0000 7fc82a3e7640 1 -- 172.21.15.78:0/215289945 shutdown_connections 2024-09-21T09:22:35.486 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.480+0000 7fc82a3e7640 1 --2- 172.21.15.78:0/215289945 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fc82412b880 0x7fc82412bcb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:35.486 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.480+0000 7fc82a3e7640 1 --2- 172.21.15.78:0/215289945 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fc82406f7e0 0x7fc82406fbc0 unknown :-1 s=CLOSED pgs=108 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:35.486 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.480+0000 7fc82a3e7640 1 -- 172.21.15.78:0/215289945 >> 172.21.15.78:0/215289945 conn(0x7fc82409dde0 msgr2=0x7fc82409e1f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:35.486 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.480+0000 7ff36c351640 1 -- 172.21.15.78:0/3719655289 wait complete. 2024-09-21T09:22:35.487 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.480+0000 7fc82a3e7640 1 -- 172.21.15.78:0/215289945 shutdown_connections 2024-09-21T09:22:35.487 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.480+0000 7fc82a3e7640 1 -- 172.21.15.78:0/215289945 wait complete. 2024-09-21T09:22:35.487 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.480+0000 7fb946e1d640 1 Processor -- start 2024-09-21T09:22:35.487 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.480+0000 7fb946e1d640 1 -- start start 2024-09-21T09:22:35.488 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.480+0000 7f1efc33f640 1 Processor -- start 2024-09-21T09:22:35.488 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.480+0000 7f1efc33f640 1 -- start start 2024-09-21T09:22:35.488 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.480+0000 7fb946e1d640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fb940095d40 0x7fb94012b270 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:35.488 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.480+0000 7f1efc33f640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f1ef4095f70 0x7f1ef4125b40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:35.489 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.480+0000 7fc82a3e7640 1 Processor -- start 2024-09-21T09:22:35.489 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.480+0000 7f1efc33f640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f1ef4096890 0x7f1ef4126080 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:35.489 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.480+0000 7f1efc33f640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f1ef4127720 0x7f1ef41265c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:35.489 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.480+0000 7f1efc33f640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7f1ef4134630 con 0x7f1ef4095f70 2024-09-21T09:22:35.489 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.480+0000 7f1efc33f640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7f1ef4128070 con 0x7f1ef4096890 2024-09-21T09:22:35.489 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.480+0000 7f1efc33f640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7f1ef41347b0 con 0x7f1ef4127720 2024-09-21T09:22:35.489 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.480+0000 7ff36c351640 1 Processor -- start 2024-09-21T09:22:35.490 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.480+0000 7fb946e1d640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fb94012e910 0x7fb94012b7b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:35.490 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.481+0000 7fb946e1d640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fb940131240 0x7fb94012bcf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:35.490 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.481+0000 7fb946e1d640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7fb9401344f0 con 0x7fb940131240 2024-09-21T09:22:35.490 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.481+0000 7fc82a3e7640 1 -- start start 2024-09-21T09:22:35.491 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.481+0000 7fb946e1d640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7fb940134370 con 0x7fb940095d40 2024-09-21T09:22:35.491 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.481+0000 7fb946e1d640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7fb940134670 con 0x7fb94012e910 2024-09-21T09:22:35.491 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.481+0000 7fb944bb9640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fb940095d40 0x7fb94012b270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:35.491 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.481+0000 7fb944bb9640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fb940095d40 0x7fb94012b270 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3301/0 says I am v2:172.21.15.78:37408/0 (socket says 172.21.15.78:37408) 2024-09-21T09:22:35.491 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.481+0000 7fb944bb9640 1 -- 172.21.15.78:0/2585502177 learned_addr learned my addr 172.21.15.78:0/2585502177 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:35.491 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.481+0000 7fb9453ba640 1 --2- 172.21.15.78:0/2585502177 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fb940131240 0x7fb94012bcf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:35.492 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.481+0000 7f1ef98da640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f1ef4096890 0x7f1ef4126080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:35.492 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.481+0000 7f1ef98da640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f1ef4096890 0x7f1ef4126080 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3301/0 says I am v2:172.21.15.78:37414/0 (socket says 172.21.15.78:37414) 2024-09-21T09:22:35.492 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.481+0000 7fb93ffff640 1 --2- 172.21.15.78:0/2585502177 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fb94012e910 0x7fb94012b7b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:35.492 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.481+0000 7f1efa8dc640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f1ef4127720 0x7f1ef41265c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:35.493 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.481+0000 7f1ef98da640 1 -- 172.21.15.78:0/3497399116 learned_addr learned my addr 172.21.15.78:0/3497399116 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:35.493 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.481+0000 7f1efa0db640 1 --2- 172.21.15.78:0/3497399116 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f1ef4095f70 0x7f1ef4125b40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:35.493 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.481+0000 7f1ef98da640 1 -- 172.21.15.78:0/3497399116 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f1ef4127720 msgr2=0x7f1ef41265c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:35.493 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.481+0000 7fc82a3e7640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fc82406f7e0 0x7fc82412c1f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:35.493 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.481+0000 7f1ef98da640 1 --2- 172.21.15.78:0/3497399116 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f1ef4127720 0x7f1ef41265c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:35.494 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.481+0000 7fb9453ba640 1 -- 172.21.15.78:0/2585502177 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fb94012e910 msgr2=0x7fb94012b7b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:35.494 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.481+0000 7fb9453ba640 1 --2- 172.21.15.78:0/2585502177 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fb94012e910 0x7fb94012b7b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:35.494 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.481+0000 7fb9453ba640 1 -- 172.21.15.78:0/2585502177 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fb940095d40 msgr2=0x7fb94012b270 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:35.494 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.481+0000 7fb9453ba640 1 --2- 172.21.15.78:0/2585502177 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fb940095d40 0x7fb94012b270 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:35.494 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.481+0000 7f1ef98da640 1 -- 172.21.15.78:0/3497399116 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f1ef4095f70 msgr2=0x7f1ef4125b40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:35.494 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.481+0000 7fb9453ba640 1 -- 172.21.15.78:0/2585502177 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fb940098610 con 0x7fb940131240 2024-09-21T09:22:35.495 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.481+0000 7ff36c351640 1 -- start start 2024-09-21T09:22:35.495 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.481+0000 7ff36c351640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7ff364098d40 0x7ff3640711b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:35.495 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.481+0000 7ff36c351640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7ff364099660 0x7ff3640716f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:35.495 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.481+0000 7ff36c351640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7ff36412f1c0 0x7ff364071c30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:35.495 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.481+0000 7ff36c351640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7ff364137960 con 0x7ff364099660 2024-09-21T09:22:35.495 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.481+0000 7fc82a3e7640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fc824070100 0x7fc82412c730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:35.496 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.481+0000 7ff36c351640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7ff3641377e0 con 0x7ff36412f1c0 2024-09-21T09:22:35.496 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.481+0000 7ff36c351640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7ff364137ae0 con 0x7ff364098d40 2024-09-21T09:22:35.496 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.481+0000 7f1ef98da640 1 --2- 172.21.15.78:0/3497399116 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f1ef4095f70 0x7f1ef4125b40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:35.496 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.481+0000 7fc82a3e7640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fc82412b880 0x7fc82412cc70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:35.496 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.482+0000 7ff36a8ee640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7ff36412f1c0 0x7ff364071c30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:35.497 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.481+0000 7f1ef98da640 1 -- 172.21.15.78:0/3497399116 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f1ef4123bf0 con 0x7f1ef4096890 2024-09-21T09:22:35.497 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.482+0000 7ff36a8ee640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7ff36412f1c0 0x7ff364071c30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3301/0 says I am v2:172.21.15.78:37422/0 (socket says 172.21.15.78:37422) 2024-09-21T09:22:35.497 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.482+0000 7fc82a3e7640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7fc824133fa0 con 0x7fc824070100 2024-09-21T09:22:35.497 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.482+0000 7ff36a8ee640 1 -- 172.21.15.78:0/1739606032 learned_addr learned my addr 172.21.15.78:0/1739606032 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:35.497 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.482+0000 7ff3698ec640 1 --2- 172.21.15.78:0/1739606032 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7ff364099660 0x7ff3640716f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:35.498 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.482+0000 7f1ef98da640 1 --2- 172.21.15.78:0/3497399116 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f1ef4096890 0x7f1ef4126080 secure :-1 s=READY pgs=46 cs=0 l=1 rev1=1 crypto rx=0x7f1ee40099f0 tx=0x7f1ee4005770 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:35.498 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.482+0000 7fc82a3e7640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7fc824133e20 con 0x7fc82406f7e0 2024-09-21T09:22:35.498 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.482+0000 7ff36a0ed640 1 --2- 172.21.15.78:0/1739606032 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7ff364098d40 0x7ff3640711b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:35.499 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.482+0000 7fc82a3e7640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7fc824134120 con 0x7fc82412b880 2024-09-21T09:22:35.499 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.482+0000 7fc823fff640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fc82406f7e0 0x7fc82412c1f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:35.499 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.482+0000 7fc823fff640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fc82406f7e0 0x7fc82412c1f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3301/0 says I am v2:172.21.15.78:37432/0 (socket says 172.21.15.78:37432) 2024-09-21T09:22:35.499 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.482+0000 7fc823fff640 1 -- 172.21.15.78:0/1505434926 learned_addr learned my addr 172.21.15.78:0/1505434926 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:35.499 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.482+0000 7fc828984640 1 --2- 172.21.15.78:0/1505434926 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fc82412b880 0x7fc82412cc70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:35.499 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.482+0000 7fc8237fe640 1 --2- 172.21.15.78:0/1505434926 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fc824070100 0x7fc82412c730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:35.499 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.482+0000 7f1eeaffd640 1 -- 172.21.15.78:0/3497399116 <== mon.1 v2:172.21.15.78:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1ee4043070 con 0x7f1ef4096890 2024-09-21T09:22:35.499 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.482+0000 7fb9453ba640 1 --2- 172.21.15.78:0/2585502177 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fb940131240 0x7fb94012bcf0 secure :-1 s=READY pgs=109 cs=0 l=1 rev1=1 crypto rx=0x7fb938032000 tx=0x7fb938003f10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:35.500 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.482+0000 7fb93d7fa640 1 -- 172.21.15.78:0/2585502177 <== mon.0 v2:172.21.15.78:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb938004240 con 0x7fb940131240 2024-09-21T09:22:35.500 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.482+0000 7ff36a8ee640 1 -- 172.21.15.78:0/1739606032 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7ff364098d40 msgr2=0x7ff3640711b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:35.500 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.482+0000 7ff36a8ee640 1 --2- 172.21.15.78:0/1739606032 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7ff364098d40 0x7ff3640711b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:35.500 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.482+0000 7ff36a8ee640 1 -- 172.21.15.78:0/1739606032 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7ff364099660 msgr2=0x7ff3640716f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:35.500 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.482+0000 7f1eeaffd640 1 -- 172.21.15.78:0/3497399116 <== mon.1 v2:172.21.15.78:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f1ee403f970 con 0x7f1ef4096890 2024-09-21T09:22:35.500 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.482+0000 7f1eeaffd640 1 -- 172.21.15.78:0/3497399116 <== mon.1 v2:172.21.15.78:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1ee4035030 con 0x7f1ef4096890 2024-09-21T09:22:35.500 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.482+0000 7f1efc33f640 1 -- 172.21.15.78:0/3497399116 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f1ef4126b00 con 0x7f1ef4096890 2024-09-21T09:22:35.500 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.482+0000 7f1efc33f640 1 -- 172.21.15.78:0/3497399116 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7f1ef4133f50 con 0x7f1ef4096890 2024-09-21T09:22:35.501 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.482+0000 7fc828984640 1 -- 172.21.15.78:0/1505434926 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fc82406f7e0 msgr2=0x7fc82412c1f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:35.501 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.482+0000 7fc828984640 1 --2- 172.21.15.78:0/1505434926 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fc82406f7e0 0x7fc82412c1f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:35.501 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.482+0000 7fc828984640 1 -- 172.21.15.78:0/1505434926 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fc824070100 msgr2=0x7fc82412c730 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:35.501 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.482+0000 7fc828984640 1 --2- 172.21.15.78:0/1505434926 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fc824070100 0x7fc82412c730 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:35.501 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.482+0000 7fc828984640 1 -- 172.21.15.78:0/1505434926 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fc82412efc0 con 0x7fc82412b880 2024-09-21T09:22:35.501 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.482+0000 7ff36a8ee640 1 --2- 172.21.15.78:0/1739606032 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7ff364099660 0x7ff3640716f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:35.501 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.482+0000 7ff36a8ee640 1 -- 172.21.15.78:0/1739606032 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ff364073c20 con 0x7ff36412f1c0 2024-09-21T09:22:35.502 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.482+0000 7fc828984640 1 --2- 172.21.15.78:0/1505434926 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fc82412b880 0x7fc82412cc70 secure :-1 s=READY pgs=47 cs=0 l=1 rev1=1 crypto rx=0x7fc81c00c5f0 tx=0x7fc81c00cab0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:35.502 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.482+0000 7ff36a0ed640 1 --2- 172.21.15.78:0/1739606032 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7ff364098d40 0x7ff3640711b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-09-21T09:22:35.502 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.482+0000 7fb93d7fa640 1 -- 172.21.15.78:0/2585502177 <== mon.0 v2:172.21.15.78:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fb938004400 con 0x7fb940131240 2024-09-21T09:22:35.503 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.482+0000 7ff3698ec640 1 --2- 172.21.15.78:0/1739606032 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7ff364099660 0x7ff3640716f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:22:35.503 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.483+0000 7ff36a8ee640 1 --2- 172.21.15.78:0/1739606032 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7ff36412f1c0 0x7ff364071c30 secure :-1 s=READY pgs=47 cs=0 l=1 rev1=1 crypto rx=0x7ff35c006010 tx=0x7ff35c003f10 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:35.503 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.483+0000 7fc820ff9640 1 -- 172.21.15.78:0/1505434926 <== mon.2 v2:172.21.15.78:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc81c004b20 con 0x7fc82412b880 2024-09-21T09:22:35.503 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.483+0000 7fc820ff9640 1 -- 172.21.15.78:0/1505434926 <== mon.2 v2:172.21.15.78:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fc81c004db0 con 0x7fc82412b880 2024-09-21T09:22:35.503 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.483+0000 7fc82a3e7640 1 -- 172.21.15.78:0/1505434926 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fc82412ecb0 con 0x7fc82412b880 2024-09-21T09:22:35.503 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.482+0000 7fb93d7fa640 1 -- 172.21.15.78:0/2585502177 <== mon.0 v2:172.21.15.78:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb938004850 con 0x7fb940131240 2024-09-21T09:22:35.504 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.482+0000 7fb946e1d640 1 -- 172.21.15.78:0/2585502177 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fb9400719a0 con 0x7fb940131240 2024-09-21T09:22:35.504 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.483+0000 7ff35affd640 1 -- 172.21.15.78:0/1739606032 <== mon.1 v2:172.21.15.78:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff35c041020 con 0x7ff36412f1c0 2024-09-21T09:22:35.504 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.483+0000 7fc820ff9640 1 -- 172.21.15.78:0/1505434926 <== mon.2 v2:172.21.15.78:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc81c003bd0 con 0x7fc82412b880 2024-09-21T09:22:35.504 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.483+0000 7fb946e1d640 1 -- 172.21.15.78:0/2585502177 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fb94012c230 con 0x7fb940131240 2024-09-21T09:22:35.505 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.483+0000 7fc82a3e7640 1 -- 172.21.15.78:0/1505434926 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7fc8241286f0 con 0x7fc82412b880 2024-09-21T09:22:35.505 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.483+0000 7ff36c351640 1 -- 172.21.15.78:0/1739606032 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7ff3640737b0 con 0x7ff36412f1c0 2024-09-21T09:22:35.505 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.483+0000 7ff36c351640 1 -- 172.21.15.78:0/1739606032 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7ff3640733e0 con 0x7ff36412f1c0 2024-09-21T09:22:35.505 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.484+0000 7ff35affd640 1 -- 172.21.15.78:0/1739606032 <== mon.1 v2:172.21.15.78:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7ff35c004330 con 0x7ff36412f1c0 2024-09-21T09:22:35.505 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.484+0000 7ff35affd640 1 -- 172.21.15.78:0/1739606032 <== mon.1 v2:172.21.15.78:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff35c036040 con 0x7ff36412f1c0 2024-09-21T09:22:35.505 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.484+0000 7fc82a3e7640 1 -- 172.21.15.78:0/1505434926 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_get_version(what=osdmap handle=1) -- 0x7fc82406fbc0 con 0x7fc82412b880 2024-09-21T09:22:35.506 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.484+0000 7f1efc33f640 1 -- 172.21.15.78:0/3497399116 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_get_version(what=osdmap handle=1) -- 0x7f1ef40749b0 con 0x7f1ef4096890 2024-09-21T09:22:35.506 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.484+0000 7fc820ff9640 1 -- 172.21.15.78:0/1505434926 <== mon.2 v2:172.21.15.78:3302/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7fc81c01d020 con 0x7fc82412b880 2024-09-21T09:22:35.506 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.484+0000 7fb93d7fa640 1 -- 172.21.15.78:0/2585502177 <== mon.0 v2:172.21.15.78:3300/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7fb938031450 con 0x7fb940131240 2024-09-21T09:22:35.506 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.484+0000 7fb946e1d640 1 -- 172.21.15.78:0/2585502177 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_get_version(what=osdmap handle=1) -- 0x7fb940098d80 con 0x7fb940131240 2024-09-21T09:22:35.507 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.485+0000 7fb93d7fa640 1 -- 172.21.15.78:0/2585502177 <== mon.0 v2:172.21.15.78:3300/0 5 ==== osd_map(32..32 src has 1..32) ==== 3027+0+0 (secure 0 0 0) 0x7fb93808a810 con 0x7fb940131240 2024-09-21T09:22:35.507 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.485+0000 7fb93d7fa640 1 -- 172.21.15.78:0/2585502177 --> v1:172.21.15.78:6800/2538472627 -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7fb938050e50 con 0x7fb924057fd0 2024-09-21T09:22:35.507 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.485+0000 7ff36c351640 1 -- 172.21.15.78:0/1739606032 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_get_version(what=osdmap handle=1) -- 0x7ff34c001030 con 0x7ff36412f1c0 2024-09-21T09:22:35.507 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.485+0000 7fc820ff9640 1 -- 172.21.15.78:0/1505434926 <== mon.2 v2:172.21.15.78:3302/0 5 ==== osd_map(32..32 src has 1..32) ==== 3027+0+0 (secure 0 0 0) 0x7fc81c069530 con 0x7fc82412b880 2024-09-21T09:22:35.507 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.485+0000 7fc820ff9640 1 -- 172.21.15.78:0/1505434926 --> v1:172.21.15.78:6812/3499964365 -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7fc81c0020b0 con 0x7fc810057e20 2024-09-21T09:22:35.508 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.485+0000 7f1eeaffd640 1 -- 172.21.15.78:0/3497399116 <== mon.1 v2:172.21.15.78:3301/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f1ee4046070 con 0x7f1ef4096890 2024-09-21T09:22:35.508 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.485+0000 7fb93d7fa640 1 -- 172.21.15.78:0/2585502177 <== mon.0 v2:172.21.15.78:3300/0 6 ==== mon_get_version_reply(handle=1 version=32) ==== 24+0+0 (secure 0 0 0) 0x7fb938066f60 con 0x7fb940131240 2024-09-21T09:22:35.508 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.486+0000 7ff35affd640 1 -- 172.21.15.78:0/1739606032 <== mon.1 v2:172.21.15.78:3301/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7ff35c044020 con 0x7ff36412f1c0 2024-09-21T09:22:35.508 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.486+0000 7f1eeaffd640 1 -- 172.21.15.78:0/3497399116 <== mon.1 v2:172.21.15.78:3301/0 5 ==== osd_map(32..32 src has 1..32) ==== 3027+0+0 (secure 0 0 0) 0x7f1ee408e710 con 0x7f1ef4096890 2024-09-21T09:22:35.509 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.486+0000 7f1eeaffd640 1 -- 172.21.15.78:0/3497399116 --> v1:172.21.15.78:6808/2733362473 -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7f1ee402eec0 con 0x7f1ed8057d30 2024-09-21T09:22:35.509 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.486+0000 7ff35affd640 1 -- 172.21.15.78:0/1739606032 <== mon.1 v2:172.21.15.78:3301/0 5 ==== osd_map(32..32 src has 1..32) ==== 3027+0+0 (secure 0 0 0) 0x7ff35c089f20 con 0x7ff36412f1c0 2024-09-21T09:22:35.509 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.486+0000 7fb93d7fa640 1 -- 172.21.15.78:0/2585502177 <== osd.0 v1:172.21.15.78:6800/2538472627 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (unknown 0 0 1302949928) 0x7fb938050e50 con 0x7fb924057fd0 2024-09-21T09:22:35.509 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.486+0000 7ff35affd640 1 -- 172.21.15.78:0/1739606032 --> v1:172.21.15.78:6802/3598780309 -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7ff35c02dec0 con 0x7ff348057e00 2024-09-21T09:22:35.510 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.487+0000 7fc820ff9640 1 -- 172.21.15.78:0/1505434926 <== osd.3 v1:172.21.15.78:6812/3499964365 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (unknown 0 0 1302949928) 0x7fc81c0020b0 con 0x7fc810057e20 2024-09-21T09:22:35.510 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.487+0000 7f1eeaffd640 1 -- 172.21.15.78:0/3497399116 <== osd.2 v1:172.21.15.78:6808/2733362473 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (unknown 0 0 1302949928) 0x7f1ee402eec0 con 0x7f1ed8057d30 2024-09-21T09:22:35.510 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.488+0000 7ff35affd640 1 -- 172.21.15.78:0/1739606032 <== osd.1 v1:172.21.15.78:6802/3598780309 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (unknown 0 0 1302949928) 0x7ff35c02dec0 con 0x7ff348057e00 2024-09-21T09:22:35.510 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.489+0000 7fc820ff9640 1 -- 172.21.15.78:0/1505434926 <== mon.2 v2:172.21.15.78:3302/0 6 ==== mon_get_version_reply(handle=1 version=32) ==== 24+0+0 (secure 0 0 0) 0x7fc81c070050 con 0x7fc82412b880 2024-09-21T09:22:35.511 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.490+0000 7f1eeaffd640 1 -- 172.21.15.78:0/3497399116 <== mon.1 v2:172.21.15.78:3301/0 6 ==== mon_get_version_reply(handle=1 version=32) ==== 24+0+0 (secure 0 0 0) 0x7f1ee4095050 con 0x7f1ef4096890 2024-09-21T09:22:35.511 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.490+0000 7ff35affd640 1 -- 172.21.15.78:0/1739606032 <== mon.1 v2:172.21.15.78:3301/0 6 ==== mon_get_version_reply(handle=1 version=32) ==== 24+0+0 (secure 0 0 0) 0x7ff35c062ea0 con 0x7ff36412f1c0 2024-09-21T09:22:35.511 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.501+0000 7fb946e1d640 1 -- 172.21.15.78:0/2585502177 --> v1:172.21.15.78:6800/2538472627 -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7fb940132880 con 0x7fb924057fd0 2024-09-21T09:22:35.511 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.502+0000 7fb93d7fa640 1 -- 172.21.15.78:0/2585502177 <== osd.0 v1:172.21.15.78:6800/2538472627 2 ==== command_reply(tid 2: 0 ) ==== 8+0+12 (unknown 0 0 795413372) 0x7fb940132880 con 0x7fb924057fd0 2024-09-21T09:22:35.512 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.502+0000 7fb946e1d640 1 -- 172.21.15.78:0/2585502177 >> v1:172.21.15.78:6800/2538472627 conn(0x7fb924057fd0 legacy=0x7fb92405a410 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:35.512 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.502+0000 7fb946e1d640 1 -- 172.21.15.78:0/2585502177 >> v1:172.21.15.78:6816/2186157269 conn(0x7fb924051750 legacy=0x7fb924053c10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:35.512 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.502+0000 7fb946e1d640 1 -- 172.21.15.78:0/2585502177 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fb940131240 msgr2=0x7fb94012bcf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:35.513 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.502+0000 7fb946e1d640 1 --2- 172.21.15.78:0/2585502177 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fb940131240 0x7fb94012bcf0 secure :-1 s=READY pgs=109 cs=0 l=1 rev1=1 crypto rx=0x7fb938032000 tx=0x7fb938003f10 comp rx=0 tx=0).stop 2024-09-21T09:22:35.513 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.502+0000 7fb9453ba640 1 -- 172.21.15.78:0/2585502177 reap_dead start 2024-09-21T09:22:35.513 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.502+0000 7fb946e1d640 1 -- 172.21.15.78:0/2585502177 shutdown_connections 2024-09-21T09:22:35.513 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.502+0000 7fb946e1d640 1 -- 172.21.15.78:0/2585502177 >> 172.21.15.78:0/2585502177 conn(0x7fb94009c230 msgr2=0x7fb940099ab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:35.513 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.502+0000 7fb946e1d640 1 -- 172.21.15.78:0/2585502177 shutdown_connections 2024-09-21T09:22:35.513 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.502+0000 7fb946e1d640 1 -- 172.21.15.78:0/2585502177 wait complete. 2024-09-21T09:22:35.513 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.503+0000 7ff36c351640 1 -- 172.21.15.78:0/1739606032 --> v1:172.21.15.78:6802/3598780309 -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7ff34c002dd0 con 0x7ff348057e00 2024-09-21T09:22:35.513 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.503+0000 7ff35affd640 1 -- 172.21.15.78:0/1739606032 <== osd.1 v1:172.21.15.78:6802/3598780309 2 ==== command_reply(tid 2: 0 ) ==== 8+0+12 (unknown 0 0 795413372) 0x7ff34c002dd0 con 0x7ff348057e00 2024-09-21T09:22:35.514 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.503+0000 7ff36c351640 1 -- 172.21.15.78:0/1739606032 >> v1:172.21.15.78:6802/3598780309 conn(0x7ff348057e00 legacy=0x7ff34805a240 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:35.515 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.503+0000 7ff36c351640 1 -- 172.21.15.78:0/1739606032 >> v1:172.21.15.78:6816/2186157269 conn(0x7ff348051580 legacy=0x7ff348053a40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:35.515 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.503+0000 7ff36c351640 1 -- 172.21.15.78:0/1739606032 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7ff36412f1c0 msgr2=0x7ff364071c30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:35.515 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.503+0000 7ff36c351640 1 --2- 172.21.15.78:0/1739606032 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7ff36412f1c0 0x7ff364071c30 secure :-1 s=READY pgs=47 cs=0 l=1 rev1=1 crypto rx=0x7ff35c006010 tx=0x7ff35c003f10 comp rx=0 tx=0).stop 2024-09-21T09:22:35.515 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.503+0000 7ff36a8ee640 1 -- 172.21.15.78:0/1739606032 reap_dead start 2024-09-21T09:22:35.515 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.503+0000 7ff36c351640 1 -- 172.21.15.78:0/1739606032 shutdown_connections 2024-09-21T09:22:35.515 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.503+0000 7ff36c351640 1 -- 172.21.15.78:0/1739606032 >> 172.21.15.78:0/1739606032 conn(0x7ff3640a1790 msgr2=0x7ff364128440 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:35.515 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.504+0000 7ff36c351640 1 -- 172.21.15.78:0/1739606032 shutdown_connections 2024-09-21T09:22:35.515 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.504+0000 7ff36c351640 1 -- 172.21.15.78:0/1739606032 wait complete. 2024-09-21T09:22:35.516 INFO:teuthology.orchestra.run.smithi078.stdout:111669149699 2024-09-21T09:22:35.516 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.0 2024-09-21T09:22:35.517 INFO:teuthology.orchestra.run.smithi078.stdout:111669149699 2024-09-21T09:22:35.518 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.1 2024-09-21T09:22:35.520 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.518+0000 7fc82a3e7640 1 -- 172.21.15.78:0/1505434926 --> v1:172.21.15.78:6812/3499964365 -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7fc82412e560 con 0x7fc810057e20 2024-09-21T09:22:35.522 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.518+0000 7f1efc33f640 1 -- 172.21.15.78:0/3497399116 --> v1:172.21.15.78:6808/2733362473 -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7f1ef4134270 con 0x7f1ed8057d30 2024-09-21T09:22:35.522 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.519+0000 7f1eeaffd640 1 -- 172.21.15.78:0/3497399116 <== osd.2 v1:172.21.15.78:6808/2733362473 2 ==== command_reply(tid 2: 0 ) ==== 8+0+12 (unknown 0 0 3324758384) 0x7f1ef4134270 con 0x7f1ed8057d30 2024-09-21T09:22:35.522 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.519+0000 7f1efc33f640 1 -- 172.21.15.78:0/3497399116 >> v1:172.21.15.78:6808/2733362473 conn(0x7f1ed8057d30 legacy=0x7f1ed805a170 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:35.522 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.519+0000 7f1efc33f640 1 -- 172.21.15.78:0/3497399116 >> v1:172.21.15.78:6816/2186157269 conn(0x7f1ed80514b0 legacy=0x7f1ed8053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:35.523 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.519+0000 7f1efc33f640 1 -- 172.21.15.78:0/3497399116 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f1ef4096890 msgr2=0x7f1ef4126080 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:35.523 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.519+0000 7f1efc33f640 1 --2- 172.21.15.78:0/3497399116 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f1ef4096890 0x7f1ef4126080 secure :-1 s=READY pgs=46 cs=0 l=1 rev1=1 crypto rx=0x7f1ee40099f0 tx=0x7f1ee4005770 comp rx=0 tx=0).stop 2024-09-21T09:22:35.523 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.519+0000 7f1efa8dc640 1 -- 172.21.15.78:0/3497399116 reap_dead start 2024-09-21T09:22:35.523 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.519+0000 7f1efc33f640 1 -- 172.21.15.78:0/3497399116 shutdown_connections 2024-09-21T09:22:35.523 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.519+0000 7f1efc33f640 1 -- 172.21.15.78:0/3497399116 >> 172.21.15.78:0/3497399116 conn(0x7f1ef409afd0 msgr2=0x7f1ef40981d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:35.523 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.518+0000 7fc820ff9640 1 -- 172.21.15.78:0/1505434926 <== osd.3 v1:172.21.15.78:6812/3499964365 2 ==== command_reply(tid 2: 0 ) ==== 8+0+12 (unknown 0 0 3556540785) 0x7fc82412e560 con 0x7fc810057e20 2024-09-21T09:22:35.523 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.519+0000 7fc82a3e7640 1 -- 172.21.15.78:0/1505434926 >> v1:172.21.15.78:6812/3499964365 conn(0x7fc810057e20 legacy=0x7fc81005a260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:35.523 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.519+0000 7fc82a3e7640 1 -- 172.21.15.78:0/1505434926 >> v1:172.21.15.78:6816/2186157269 conn(0x7fc810051530 legacy=0x7fc8100539f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:35.523 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.519+0000 7fc82a3e7640 1 -- 172.21.15.78:0/1505434926 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fc82412b880 msgr2=0x7fc82412cc70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:35.524 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.519+0000 7fc82a3e7640 1 --2- 172.21.15.78:0/1505434926 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fc82412b880 0x7fc82412cc70 secure :-1 s=READY pgs=47 cs=0 l=1 rev1=1 crypto rx=0x7fc81c00c5f0 tx=0x7fc81c00cab0 comp rx=0 tx=0).stop 2024-09-21T09:22:35.524 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.519+0000 7fc828984640 1 -- 172.21.15.78:0/1505434926 reap_dead start 2024-09-21T09:22:35.524 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.519+0000 7fc82a3e7640 1 -- 172.21.15.78:0/1505434926 shutdown_connections 2024-09-21T09:22:35.524 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.519+0000 7fc82a3e7640 1 -- 172.21.15.78:0/1505434926 >> 172.21.15.78:0/1505434926 conn(0x7fc82409dde0 msgr2=0x7fc824124a90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:35.524 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.519+0000 7f1efc33f640 1 -- 172.21.15.78:0/3497399116 shutdown_connections 2024-09-21T09:22:35.524 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.519+0000 7f1efc33f640 1 -- 172.21.15.78:0/3497399116 wait complete. 2024-09-21T09:22:35.524 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.519+0000 7fc82a3e7640 1 -- 172.21.15.78:0/1505434926 shutdown_connections 2024-09-21T09:22:35.525 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.519+0000 7fc82a3e7640 1 -- 172.21.15.78:0/1505434926 wait complete. 2024-09-21T09:22:35.534 INFO:teuthology.orchestra.run.smithi078.stdout:120259084291 2024-09-21T09:22:35.535 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.3 2024-09-21T09:22:35.535 INFO:teuthology.orchestra.run.smithi078.stdout:115964116995 2024-09-21T09:22:35.536 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.2 2024-09-21T09:22:35.626 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.623+0000 7f92888ef640 1 -- 172.21.15.78:0/3247853854 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f9284125b70 msgr2=0x7f9284125f50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:35.626 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.623+0000 7f92888ef640 1 --2- 172.21.15.78:0/3247853854 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f9284125b70 0x7f9284125f50 secure :-1 s=READY pgs=48 cs=0 l=1 rev1=1 crypto rx=0x7f9274009830 tx=0x7f927402a3f0 comp rx=0 tx=0).stop 2024-09-21T09:22:35.626 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.623+0000 7f92888ef640 1 -- 172.21.15.78:0/3247853854 shutdown_connections 2024-09-21T09:22:35.626 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.623+0000 7f92888ef640 1 --2- 172.21.15.78:0/3247853854 >> v2:172.21.15.78:3300/0 conn(0x7f9284095c50 0x7f9284096070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:35.626 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.623+0000 7f92888ef640 1 --2- 172.21.15.78:0/3247853854 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f9284125b70 0x7f9284125f50 unknown :-1 s=CLOSED pgs=48 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:35.627 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.623+0000 7f92888ef640 1 -- 172.21.15.78:0/3247853854 >> 172.21.15.78:0/3247853854 conn(0x7f928409e610 msgr2=0x7f928409ea20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:35.627 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.624+0000 7f92888ef640 1 -- 172.21.15.78:0/3247853854 shutdown_connections 2024-09-21T09:22:35.627 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.624+0000 7f92888ef640 1 -- 172.21.15.78:0/3247853854 wait complete. 2024-09-21T09:22:35.627 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.624+0000 7f16e6c52640 1 -- 172.21.15.78:0/2353699254 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f16e012f1c0 msgr2=0x7f16e012f600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:35.628 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.624+0000 7f16e6c52640 1 --2- 172.21.15.78:0/2353699254 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f16e012f1c0 0x7f16e012f600 secure :-1 s=READY pgs=48 cs=0 l=1 rev1=1 crypto rx=0x7f16d8005fd0 tx=0x7f16d802a780 comp rx=0 tx=0).stop 2024-09-21T09:22:35.628 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.624+0000 7f16e6c52640 1 -- 172.21.15.78:0/2353699254 shutdown_connections 2024-09-21T09:22:35.628 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.624+0000 7f16e6c52640 1 --2- 172.21.15.78:0/2353699254 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f16e012f1c0 0x7f16e012f600 unknown :-1 s=CLOSED pgs=48 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:35.628 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.624+0000 7f16e6c52640 1 --2- 172.21.15.78:0/2353699254 >> v2:172.21.15.78:3300/0 conn(0x7f16e0098d40 0x7f16e0099120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:35.628 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.624+0000 7f16e6c52640 1 -- 172.21.15.78:0/2353699254 >> 172.21.15.78:0/2353699254 conn(0x7f16e00a1790 msgr2=0x7f16e00a1ba0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:35.628 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.624+0000 7f16e6c52640 1 -- 172.21.15.78:0/2353699254 shutdown_connections 2024-09-21T09:22:35.628 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.624+0000 7f16e6c52640 1 -- 172.21.15.78:0/2353699254 wait complete. 2024-09-21T09:22:35.628 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.625+0000 7f92888ef640 1 Processor -- start 2024-09-21T09:22:35.629 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.625+0000 7f16e6c52640 1 Processor -- start 2024-09-21T09:22:35.629 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.625+0000 7f92888ef640 1 -- start start 2024-09-21T09:22:35.629 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.625+0000 7f16e6c52640 1 -- start start 2024-09-21T09:22:35.629 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.626+0000 7f16e6c52640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f16e0098d40 0x7f16e0071080 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:35.630 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.626+0000 7f92888ef640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f9284095c50 0x7f928408df40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:35.631 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.626+0000 7f16e6c52640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f16e0099660 0x7f16e00715c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:35.631 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.626+0000 7f16e6c52640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f16e012f1c0 0x7f16e0071b00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:35.631 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.626+0000 7f16e6c52640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7f16e0137870 con 0x7f16e0098d40 2024-09-21T09:22:35.631 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.626+0000 7f16e6c52640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7f16e01376f0 con 0x7f16e0099660 2024-09-21T09:22:35.631 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.626+0000 7f16e6c52640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7f16e01379f0 con 0x7f16e012f1c0 2024-09-21T09:22:35.631 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.626+0000 7f16e49ee640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f16e0098d40 0x7f16e0071080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:35.631 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.626+0000 7f16e49ee640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f16e0098d40 0x7f16e0071080 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3300/0 says I am v2:172.21.15.78:44500/0 (socket says 172.21.15.78:44500) 2024-09-21T09:22:35.631 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.626+0000 7f16e49ee640 1 -- 172.21.15.78:0/4084650021 learned_addr learned my addr 172.21.15.78:0/4084650021 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:35.632 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.626+0000 7f16d7fff640 1 --2- 172.21.15.78:0/4084650021 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f16e0099660 0x7f16e00715c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:35.632 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.626+0000 7f16e51ef640 1 --2- 172.21.15.78:0/4084650021 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f16e012f1c0 0x7f16e0071b00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:35.632 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.626+0000 7f16e49ee640 1 -- 172.21.15.78:0/4084650021 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f16e012f1c0 msgr2=0x7f16e0071b00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:35.632 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.626+0000 7f92888ef640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f92840965b0 0x7f928408e480 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:35.632 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.626+0000 7f92888ef640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f9284125b70 0x7f928408e9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:35.632 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.626+0000 7f928259c640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f9284095c50 0x7f928408df40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:35.632 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.626+0000 7f928259c640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f9284095c50 0x7f928408df40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3300/0 says I am v2:172.21.15.78:44502/0 (socket says 172.21.15.78:44502) 2024-09-21T09:22:35.632 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.626+0000 7f9281d9b640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f92840965b0 0x7f928408e480 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:35.632 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.626+0000 7f9281d9b640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f92840965b0 0x7f928408e480 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3302/0 says I am v2:172.21.15.78:35664/0 (socket says 172.21.15.78:35664) 2024-09-21T09:22:35.633 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.626+0000 7f9282d9d640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f9284125b70 0x7f928408e9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:35.633 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.626+0000 7f16e49ee640 1 --2- 172.21.15.78:0/4084650021 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f16e012f1c0 0x7f16e0071b00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:35.633 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.626+0000 7f16e49ee640 1 -- 172.21.15.78:0/4084650021 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f16e0099660 msgr2=0x7f16e00715c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:35.633 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.626+0000 7f16e49ee640 1 --2- 172.21.15.78:0/4084650021 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f16e0099660 0x7f16e00715c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:35.633 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.626+0000 7f16e49ee640 1 -- 172.21.15.78:0/4084650021 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f16e0073680 con 0x7f16e0098d40 2024-09-21T09:22:35.634 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.626+0000 7f9282d9d640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f9284125b70 0x7f928408e9c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3301/0 says I am v2:172.21.15.78:37452/0 (socket says 172.21.15.78:37452) 2024-09-21T09:22:35.634 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.626+0000 7f92888ef640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7f92841347f0 con 0x7f9284095c50 2024-09-21T09:22:35.634 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.626+0000 7f92888ef640 1 -- 172.21.15.78:0/3183412289 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7f9284134670 con 0x7f9284125b70 2024-09-21T09:22:35.634 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.626+0000 7f92888ef640 1 -- 172.21.15.78:0/3183412289 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7f9284134970 con 0x7f92840965b0 2024-09-21T09:22:35.634 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.626+0000 7f928259c640 1 -- 172.21.15.78:0/3183412289 learned_addr learned my addr 172.21.15.78:0/3183412289 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:35.634 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.626+0000 7f16e51ef640 1 --2- 172.21.15.78:0/4084650021 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f16e012f1c0 0x7f16e0071b00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-09-21T09:22:35.634 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.626+0000 7f16e49ee640 1 --2- 172.21.15.78:0/4084650021 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f16e0098d40 0x7f16e0071080 secure :-1 s=READY pgs=112 cs=0 l=1 rev1=1 crypto rx=0x7f16dc00c970 tx=0x7f16dc00ce30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:35.635 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.627+0000 7f928259c640 1 -- 172.21.15.78:0/3183412289 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f92840965b0 msgr2=0x7f928408e480 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:35.635 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.627+0000 7f928259c640 1 --2- 172.21.15.78:0/3183412289 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f92840965b0 0x7f928408e480 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:35.635 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.627+0000 7f928259c640 1 -- 172.21.15.78:0/3183412289 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f9284125b70 msgr2=0x7f928408e9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:35.635 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.627+0000 7f16d57fa640 1 -- 172.21.15.78:0/4084650021 <== mon.0 v2:172.21.15.78:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f16dc007bb0 con 0x7f16e0098d40 2024-09-21T09:22:35.635 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.627+0000 7f16d57fa640 1 -- 172.21.15.78:0/4084650021 <== mon.0 v2:172.21.15.78:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f16dc007e40 con 0x7f16e0098d40 2024-09-21T09:22:35.636 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.627+0000 7f16d57fa640 1 -- 172.21.15.78:0/4084650021 <== mon.0 v2:172.21.15.78:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f16dc00f690 con 0x7f16e0098d40 2024-09-21T09:22:35.636 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.627+0000 7f928259c640 1 --2- 172.21.15.78:0/3183412289 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f9284125b70 0x7f928408e9c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:35.636 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.627+0000 7f16e6c52640 1 -- 172.21.15.78:0/4084650021 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f16e0136740 con 0x7f16e0098d40 2024-09-21T09:22:35.636 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.627+0000 7f16e6c52640 1 -- 172.21.15.78:0/4084650021 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f16e0073d30 con 0x7f16e0098d40 2024-09-21T09:22:35.637 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.627+0000 7f928259c640 1 -- 172.21.15.78:0/3183412289 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f92840912f0 con 0x7f9284095c50 2024-09-21T09:22:35.637 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.627+0000 7f928259c640 1 --2- 172.21.15.78:0/3183412289 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f9284095c50 0x7f928408df40 secure :-1 s=READY pgs=113 cs=0 l=1 rev1=1 crypto rx=0x7f9274009950 tx=0x7f9274032c10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:35.637 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.627+0000 7f9282d9d640 1 --2- 172.21.15.78:0/3183412289 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f9284125b70 0x7f928408e9c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:22:35.637 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.628+0000 7f9272ffd640 1 -- 172.21.15.78:0/3183412289 <== mon.0 v2:172.21.15.78:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9274040070 con 0x7f9284095c50 2024-09-21T09:22:35.637 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.628+0000 7f9272ffd640 1 -- 172.21.15.78:0/3183412289 <== mon.0 v2:172.21.15.78:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f9274005310 con 0x7f9284095c50 2024-09-21T09:22:35.637 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.628+0000 7f9272ffd640 1 -- 172.21.15.78:0/3183412289 <== mon.0 v2:172.21.15.78:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9274005c90 con 0x7f9284095c50 2024-09-21T09:22:35.637 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.628+0000 7f92888ef640 1 -- 172.21.15.78:0/3183412289 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f928408ef00 con 0x7f9284095c50 2024-09-21T09:22:35.637 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.628+0000 7f92888ef640 1 -- 172.21.15.78:0/3183412289 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f9284090740 con 0x7f9284095c50 2024-09-21T09:22:35.638 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.628+0000 7f16e6c52640 1 -- 172.21.15.78:0/4084650021 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f16e0099120 con 0x7f16e0098d40 2024-09-21T09:22:35.638 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.628+0000 7f16d57fa640 1 -- 172.21.15.78:0/4084650021 <== mon.0 v2:172.21.15.78:3300/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f16dc018400 con 0x7f16e0098d40 2024-09-21T09:22:35.638 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.629+0000 7f16d57fa640 1 -- 172.21.15.78:0/4084650021 <== mon.0 v2:172.21.15.78:3300/0 5 ==== osd_map(32..32 src has 1..32) ==== 3027+0+0 (secure 0 0 0) 0x7f16dc0696b0 con 0x7f16e0098d40 2024-09-21T09:22:35.639 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.629+0000 7f9272ffd640 1 -- 172.21.15.78:0/3183412289 <== mon.0 v2:172.21.15.78:3300/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f9274039040 con 0x7f9284095c50 2024-09-21T09:22:35.639 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.629+0000 7f92888ef640 1 -- 172.21.15.78:0/3183412289 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f928409cf90 con 0x7f9284095c50 2024-09-21T09:22:35.639 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.630+0000 7f9272ffd640 1 -- 172.21.15.78:0/3183412289 <== mon.0 v2:172.21.15.78:3300/0 5 ==== osd_map(32..32 src has 1..32) ==== 3027+0+0 (secure 0 0 0) 0x7f927408a210 con 0x7f9284095c50 2024-09-21T09:22:35.639 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.636+0000 7f16d57fa640 1 -- 172.21.15.78:0/4084650021 <== mon.0 v2:172.21.15.78:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f16dc014030 con 0x7f16e0098d40 2024-09-21T09:22:35.645 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.643+0000 7f9272ffd640 1 -- 172.21.15.78:0/3183412289 <== mon.0 v2:172.21.15.78:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f928409cf90 con 0x7f9284095c50 2024-09-21T09:22:35.686 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.684+0000 7f31cc7a9640 1 -- 172.21.15.78:0/2083524858 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f31c412bdb0 msgr2=0x7f31c4094e60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:35.686 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.684+0000 7f31cc7a9640 1 --2- 172.21.15.78:0/2083524858 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f31c412bdb0 0x7f31c4094e60 secure :-1 s=READY pgs=49 cs=0 l=1 rev1=1 crypto rx=0x7f31b4009900 tx=0x7f31b402a580 comp rx=0 tx=0).stop 2024-09-21T09:22:35.686 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.684+0000 7f31cc7a9640 1 -- 172.21.15.78:0/2083524858 shutdown_connections 2024-09-21T09:22:35.686 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.684+0000 7f31cc7a9640 1 --2- 172.21.15.78:0/2083524858 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f31c412bdb0 0x7f31c4094e60 unknown :-1 s=CLOSED pgs=49 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:35.687 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.684+0000 7f31cc7a9640 1 --2- 172.21.15.78:0/2083524858 >> v2:172.21.15.78:3300/0 conn(0x7f31c412b490 0x7f31c412b870 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:35.687 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.684+0000 7f31cc7a9640 1 -- 172.21.15.78:0/2083524858 >> 172.21.15.78:0/2083524858 conn(0x7f31c409afd0 msgr2=0x7f31c409b3e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:35.687 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.684+0000 7f31cc7a9640 1 -- 172.21.15.78:0/2083524858 shutdown_connections 2024-09-21T09:22:35.687 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.684+0000 7f31cc7a9640 1 -- 172.21.15.78:0/2083524858 wait complete. 2024-09-21T09:22:35.688 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.685+0000 7f31cc7a9640 1 Processor -- start 2024-09-21T09:22:35.688 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.685+0000 7f31cc7a9640 1 -- start start 2024-09-21T09:22:35.688 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.685+0000 7f31cc7a9640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f31c40953a0 0x7f31c412c440 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:35.688 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.685+0000 7f31cc7a9640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f31c412b490 0x7f31c412c980 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:35.688 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.685+0000 7f31ca545640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f31c40953a0 0x7f31c412c440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:35.688 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.685+0000 7f31ca545640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f31c40953a0 0x7f31c412c440 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3300/0 says I am v2:172.21.15.78:44514/0 (socket says 172.21.15.78:44514) 2024-09-21T09:22:35.688 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.685+0000 7f31cc7a9640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f31c412bdb0 0x7f31c412cec0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:35.688 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.685+0000 7f31ca545640 1 -- 172.21.15.78:0/2783935577 learned_addr learned my addr 172.21.15.78:0/2783935577 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:35.689 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.685+0000 7f31cc7a9640 1 -- 172.21.15.78:0/2783935577 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7f31c4133600 con 0x7f31c40953a0 2024-09-21T09:22:35.689 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.685+0000 7f31cc7a9640 1 -- 172.21.15.78:0/2783935577 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7f31c4133480 con 0x7f31c412bdb0 2024-09-21T09:22:35.689 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.685+0000 7f31cc7a9640 1 -- 172.21.15.78:0/2783935577 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7f31c4133780 con 0x7f31c412b490 2024-09-21T09:22:35.689 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.686+0000 7f31c9d44640 1 --2- 172.21.15.78:0/2783935577 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f31c412b490 0x7f31c412c980 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:35.689 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.686+0000 7f31cad46640 1 --2- 172.21.15.78:0/2783935577 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f31c412bdb0 0x7f31c412cec0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:35.689 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.686+0000 7f31ca545640 1 -- 172.21.15.78:0/2783935577 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f31c412b490 msgr2=0x7f31c412c980 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:35.689 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.686+0000 7f31ca545640 1 --2- 172.21.15.78:0/2783935577 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f31c412b490 0x7f31c412c980 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:35.689 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.686+0000 7f31ca545640 1 -- 172.21.15.78:0/2783935577 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f31c412bdb0 msgr2=0x7f31c412cec0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:35.689 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.686+0000 7f31ca545640 1 --2- 172.21.15.78:0/2783935577 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f31c412bdb0 0x7f31c412cec0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:35.689 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.686+0000 7f31ca545640 1 -- 172.21.15.78:0/2783935577 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f31c40743e0 con 0x7f31c40953a0 2024-09-21T09:22:35.689 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.686+0000 7f31c9d44640 1 --2- 172.21.15.78:0/2783935577 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f31c412b490 0x7f31c412c980 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:22:35.690 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.686+0000 7f31ca545640 1 --2- 172.21.15.78:0/2783935577 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f31c40953a0 0x7f31c412c440 secure :-1 s=READY pgs=116 cs=0 l=1 rev1=1 crypto rx=0x7f31c000eb20 tx=0x7f31c000c3d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:35.690 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.686+0000 7f31baffd640 1 -- 172.21.15.78:0/2783935577 <== mon.0 v2:172.21.15.78:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f31c000ce10 con 0x7f31c40953a0 2024-09-21T09:22:35.690 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.687+0000 7f31cc7a9640 1 -- 172.21.15.78:0/2783935577 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f31c4093850 con 0x7f31c40953a0 2024-09-21T09:22:35.690 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.687+0000 7f31baffd640 1 -- 172.21.15.78:0/2783935577 <== mon.0 v2:172.21.15.78:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f31c0004110 con 0x7f31c40953a0 2024-09-21T09:22:35.691 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.687+0000 7f35f5ffb640 1 -- 172.21.15.78:0/1137942495 <== mon.1 v2:172.21.15.78:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f35f0031b40 con 0x7f35f812bf30 2024-09-21T09:22:35.691 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.687+0000 7f31cc7a9640 1 -- 172.21.15.78:0/2783935577 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f31c412ea00 con 0x7f31c40953a0 2024-09-21T09:22:35.691 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.687+0000 7f31baffd640 1 -- 172.21.15.78:0/2783935577 <== mon.0 v2:172.21.15.78:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f31c00035f0 con 0x7f31c40953a0 2024-09-21T09:22:35.692 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.687+0000 7f35fd9fe640 1 -- 172.21.15.78:0/1137942495 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f35f812bf30 msgr2=0x7f35f812c3a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:35.692 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.687+0000 7f35fd9fe640 1 --2- 172.21.15.78:0/1137942495 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f35f812bf30 0x7f35f812c3a0 secure :-1 s=READY pgs=49 cs=0 l=1 rev1=1 crypto rx=0x7f35f0009be0 tx=0x7f35f002a760 comp rx=0 tx=0).stop 2024-09-21T09:22:35.692 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.688+0000 7f35fd9fe640 1 -- 172.21.15.78:0/1137942495 shutdown_connections 2024-09-21T09:22:35.692 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.688+0000 7f35fd9fe640 1 --2- 172.21.15.78:0/1137942495 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f35f812bf30 0x7f35f812c3a0 unknown :-1 s=CLOSED pgs=49 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:35.692 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.688+0000 7f35fd9fe640 1 --2- 172.21.15.78:0/1137942495 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f35f8092820 0x7f35f8092c00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:35.692 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.688+0000 7f35fd9fe640 1 -- 172.21.15.78:0/1137942495 >> 172.21.15.78:0/1137942495 conn(0x7f35f809e490 msgr2=0x7f35f809e8a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:35.692 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.688+0000 7f35fd9fe640 1 -- 172.21.15.78:0/1137942495 shutdown_connections 2024-09-21T09:22:35.692 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.688+0000 7f35fd9fe640 1 -- 172.21.15.78:0/1137942495 wait complete. 2024-09-21T09:22:35.693 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.688+0000 7f31baffd640 1 -- 172.21.15.78:0/2783935577 <== mon.0 v2:172.21.15.78:3300/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f31c001d020 con 0x7f31c40953a0 2024-09-21T09:22:35.693 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.688+0000 7f31cc7a9640 1 -- 172.21.15.78:0/2783935577 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f31c412b870 con 0x7f31c40953a0 2024-09-21T09:22:35.693 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.689+0000 7f31baffd640 1 -- 172.21.15.78:0/2783935577 <== mon.0 v2:172.21.15.78:3300/0 5 ==== osd_map(32..32 src has 1..32) ==== 3027+0+0 (secure 0 0 0) 0x7f31c00692c0 con 0x7f31c40953a0 2024-09-21T09:22:35.694 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.689+0000 7f35fd9fe640 1 Processor -- start 2024-09-21T09:22:35.694 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.689+0000 7f35fd9fe640 1 -- start start 2024-09-21T09:22:35.694 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.690+0000 7f35fd9fe640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f35f8092820 0x7f35f81c2c80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:35.694 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.690+0000 7f35fd9fe640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f35f8093140 0x7f35f81c31c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:35.694 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.690+0000 7f35fd9fe640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f35f812bf30 0x7f35f8070c60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:35.695 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.690+0000 7f35fd9fe640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7f35f8134620 con 0x7f35f812bf30 2024-09-21T09:22:35.695 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.690+0000 7f35fd9fe640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7f35f81344a0 con 0x7f35f8092820 2024-09-21T09:22:35.695 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.690+0000 7f35fd9fe640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7f35f81347a0 con 0x7f35f8093140 2024-09-21T09:22:35.695 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.690+0000 7f35f77fe640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f35f812bf30 0x7f35f8070c60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:35.695 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.690+0000 7f35f77fe640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f35f812bf30 0x7f35f8070c60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3300/0 says I am v2:172.21.15.78:44522/0 (socket says 172.21.15.78:44522) 2024-09-21T09:22:35.695 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.690+0000 7f35f77fe640 1 -- 172.21.15.78:0/3760299278 learned_addr learned my addr 172.21.15.78:0/3760299278 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:35.695 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.690+0000 7f35f6ffd640 1 --2- 172.21.15.78:0/3760299278 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f35f8092820 0x7f35f81c2c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:35.695 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.690+0000 7f35f77fe640 1 -- 172.21.15.78:0/3760299278 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f35f8093140 msgr2=0x7f35f81c31c0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-21T09:22:35.695 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.690+0000 7f35f77fe640 1 --2- 172.21.15.78:0/3760299278 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f35f8093140 0x7f35f81c31c0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:35.695 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.690+0000 7f35f77fe640 1 -- 172.21.15.78:0/3760299278 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f35f8092820 msgr2=0x7f35f81c2c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:35.695 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.690+0000 7f35f77fe640 1 --2- 172.21.15.78:0/3760299278 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f35f8092820 0x7f35f81c2c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:35.695 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.690+0000 7f35f77fe640 1 -- 172.21.15.78:0/3760299278 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f35f8072be0 con 0x7f35f812bf30 2024-09-21T09:22:35.695 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.691+0000 7f35f77fe640 1 --2- 172.21.15.78:0/3760299278 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f35f812bf30 0x7f35f8070c60 secure :-1 s=READY pgs=117 cs=0 l=1 rev1=1 crypto rx=0x7f35f0009be0 tx=0x7f35f0003c00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:35.696 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.693+0000 7f35effff640 1 -- 172.21.15.78:0/3760299278 <== mon.0 v2:172.21.15.78:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f35f0041020 con 0x7f35f812bf30 2024-09-21T09:22:35.696 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.693+0000 7f31baffd640 1 -- 172.21.15.78:0/2783935577 <== mon.0 v2:172.21.15.78:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f31c0068250 con 0x7f31c40953a0 2024-09-21T09:22:35.696 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.693+0000 7f35effff640 1 -- 172.21.15.78:0/3760299278 <== mon.0 v2:172.21.15.78:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f35f0004040 con 0x7f35f812bf30 2024-09-21T09:22:35.697 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.693+0000 7f35effff640 1 -- 172.21.15.78:0/3760299278 <== mon.0 v2:172.21.15.78:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f35f0036040 con 0x7f35f812bf30 2024-09-21T09:22:35.697 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.693+0000 7f35fd9fe640 1 -- 172.21.15.78:0/3760299278 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f35f81c3700 con 0x7f35f812bf30 2024-09-21T09:22:35.697 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.693+0000 7f35fd9fe640 1 -- 172.21.15.78:0/3760299278 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f35f80711a0 con 0x7f35f812bf30 2024-09-21T09:22:35.697 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.695+0000 7f35fd9fe640 1 -- 172.21.15.78:0/3760299278 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f35f812b300 con 0x7f35f812bf30 2024-09-21T09:22:35.697 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.695+0000 7f35effff640 1 -- 172.21.15.78:0/3760299278 <== mon.0 v2:172.21.15.78:3300/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f35f0044020 con 0x7f35f812bf30 2024-09-21T09:22:35.697 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.695+0000 7f35effff640 1 -- 172.21.15.78:0/3760299278 <== mon.0 v2:172.21.15.78:3300/0 5 ==== osd_map(32..32 src has 1..32) ==== 3027+0+0 (secure 0 0 0) 0x7f35f0089f00 con 0x7f35f812bf30 2024-09-21T09:22:35.704 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.702+0000 7f35effff640 1 -- 172.21.15.78:0/3760299278 <== mon.0 v2:172.21.15.78:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f35f812b300 con 0x7f35f812bf30 2024-09-21T09:22:35.798 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.796+0000 7f16e6c52640 1 -- 172.21.15.78:0/4084650021 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 0} v 0) -- 0x7f16e00698c0 con 0x7f16e0098d40 2024-09-21T09:22:35.799 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.797+0000 7f16d57fa640 1 -- 172.21.15.78:0/4084650021 <== mon.0 v2:172.21.15.78:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 0}]=0 v0) ==== 74+0+13 (secure 0 0 0) 0x7f16dc068000 con 0x7f16e0098d40 2024-09-21T09:22:35.799 INFO:teuthology.orchestra.run.smithi078.stdout:111669149698 2024-09-21T09:22:35.801 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.799+0000 7f16e6c52640 1 -- 172.21.15.78:0/4084650021 >> v1:172.21.15.78:6816/2186157269 conn(0x7f16cc051500 legacy=0x7f16cc0539c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:35.801 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.799+0000 7f16e6c52640 1 -- 172.21.15.78:0/4084650021 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f16e0098d40 msgr2=0x7f16e0071080 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:35.801 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.799+0000 7f16e6c52640 1 --2- 172.21.15.78:0/4084650021 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f16e0098d40 0x7f16e0071080 secure :-1 s=READY pgs=112 cs=0 l=1 rev1=1 crypto rx=0x7f16dc00c970 tx=0x7f16dc00ce30 comp rx=0 tx=0).stop 2024-09-21T09:22:35.801 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.799+0000 7f16e6c52640 1 -- 172.21.15.78:0/4084650021 shutdown_connections 2024-09-21T09:22:35.801 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.799+0000 7f16e6c52640 1 --2- 172.21.15.78:0/4084650021 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f16e012f1c0 0x7f16e0071b00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:35.802 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.799+0000 7f16e6c52640 1 --2- 172.21.15.78:0/4084650021 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f16e0099660 0x7f16e00715c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:35.802 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.799+0000 7f16e6c52640 1 --2- 172.21.15.78:0/4084650021 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f16e0098d40 0x7f16e0071080 unknown :-1 s=CLOSED pgs=112 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:35.802 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.799+0000 7f16e6c52640 1 -- 172.21.15.78:0/4084650021 >> 172.21.15.78:0/4084650021 conn(0x7f16e00a1790 msgr2=0x7f16e009ce30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:35.802 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.799+0000 7f16e6c52640 1 -- 172.21.15.78:0/4084650021 shutdown_connections 2024-09-21T09:22:35.802 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.799+0000 7f16e6c52640 1 -- 172.21.15.78:0/4084650021 wait complete. 2024-09-21T09:22:35.812 INFO:tasks.ceph.ceph_manager.ceph:need seq 111669149699 got 111669149698 for osd.0 2024-09-21T09:22:35.813 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.811+0000 7f92888ef640 1 -- 172.21.15.78:0/3183412289 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 1} v 0) -- 0x7f92840726b0 con 0x7f9284095c50 2024-09-21T09:22:35.814 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.811+0000 7f9272ffd640 1 -- 172.21.15.78:0/3183412289 <== mon.0 v2:172.21.15.78:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 1}]=0 v0) ==== 74+0+13 (secure 0 0 0) 0x7f927403fdd0 con 0x7f9284095c50 2024-09-21T09:22:35.814 INFO:teuthology.orchestra.run.smithi078.stdout:111669149698 2024-09-21T09:22:35.816 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.814+0000 7f92888ef640 1 -- 172.21.15.78:0/3183412289 >> v1:172.21.15.78:6816/2186157269 conn(0x7f9264051530 legacy=0x7f92640539f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:35.816 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.814+0000 7f92888ef640 1 -- 172.21.15.78:0/3183412289 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f9284095c50 msgr2=0x7f928408df40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:35.816 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.814+0000 7f92888ef640 1 --2- 172.21.15.78:0/3183412289 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f9284095c50 0x7f928408df40 secure :-1 s=READY pgs=113 cs=0 l=1 rev1=1 crypto rx=0x7f9274009950 tx=0x7f9274032c10 comp rx=0 tx=0).stop 2024-09-21T09:22:35.816 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.814+0000 7f92888ef640 1 -- 172.21.15.78:0/3183412289 shutdown_connections 2024-09-21T09:22:35.816 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.814+0000 7f92888ef640 1 --2- 172.21.15.78:0/3183412289 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f9284125b70 0x7f928408e9c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:35.817 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.814+0000 7f92888ef640 1 --2- 172.21.15.78:0/3183412289 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f92840965b0 0x7f928408e480 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:35.817 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.814+0000 7f92888ef640 1 --2- 172.21.15.78:0/3183412289 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f9284095c50 0x7f928408df40 unknown :-1 s=CLOSED pgs=113 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:35.817 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.814+0000 7f92888ef640 1 -- 172.21.15.78:0/3183412289 >> 172.21.15.78:0/3183412289 conn(0x7f928409e610 msgr2=0x7f928406f880 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:35.817 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.814+0000 7f92888ef640 1 -- 172.21.15.78:0/3183412289 shutdown_connections 2024-09-21T09:22:35.817 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.814+0000 7f92888ef640 1 -- 172.21.15.78:0/3183412289 wait complete. 2024-09-21T09:22:35.827 INFO:tasks.ceph.ceph_manager.ceph:need seq 111669149699 got 111669149698 for osd.1 2024-09-21T09:22:35.890 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.887+0000 7f31cc7a9640 1 -- 172.21.15.78:0/2783935577 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 3} v 0) -- 0x7f31c406fd40 con 0x7f31c40953a0 2024-09-21T09:22:35.890 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.888+0000 7f31baffd640 1 -- 172.21.15.78:0/2783935577 <== mon.0 v2:172.21.15.78:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 3}]=0 v0) ==== 74+0+2 (secure 0 0 0) 0x7f31c0067a90 con 0x7f31c40953a0 2024-09-21T09:22:35.890 INFO:teuthology.orchestra.run.smithi078.stdout:1 2024-09-21T09:22:35.892 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.889+0000 7f31cc7a9640 1 -- 172.21.15.78:0/2783935577 >> v1:172.21.15.78:6816/2186157269 conn(0x7f31a80514e0 legacy=0x7f31a80539a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:35.892 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.889+0000 7f31cc7a9640 1 -- 172.21.15.78:0/2783935577 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f31c40953a0 msgr2=0x7f31c412c440 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:35.892 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.889+0000 7f31cc7a9640 1 --2- 172.21.15.78:0/2783935577 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f31c40953a0 0x7f31c412c440 secure :-1 s=READY pgs=116 cs=0 l=1 rev1=1 crypto rx=0x7f31c000eb20 tx=0x7f31c000c3d0 comp rx=0 tx=0).stop 2024-09-21T09:22:35.892 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.889+0000 7f31cc7a9640 1 -- 172.21.15.78:0/2783935577 shutdown_connections 2024-09-21T09:22:35.892 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.889+0000 7f31cc7a9640 1 --2- 172.21.15.78:0/2783935577 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f31c412bdb0 0x7f31c412cec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:35.892 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.889+0000 7f31cc7a9640 1 --2- 172.21.15.78:0/2783935577 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f31c412b490 0x7f31c412c980 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:35.892 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.889+0000 7f31cc7a9640 1 --2- 172.21.15.78:0/2783935577 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f31c40953a0 0x7f31c412c440 unknown :-1 s=CLOSED pgs=116 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:35.893 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.889+0000 7f31cc7a9640 1 -- 172.21.15.78:0/2783935577 >> 172.21.15.78:0/2783935577 conn(0x7f31c409afd0 msgr2=0x7f31c4092da0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:35.893 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.889+0000 7f31cc7a9640 1 -- 172.21.15.78:0/2783935577 shutdown_connections 2024-09-21T09:22:35.893 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.889+0000 7f31cc7a9640 1 -- 172.21.15.78:0/2783935577 wait complete. 2024-09-21T09:22:35.895 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.893+0000 7f35fd9fe640 1 -- 172.21.15.78:0/3760299278 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 2} v 0) -- 0x7f35f8092c00 con 0x7f35f812bf30 2024-09-21T09:22:35.895 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.893+0000 7f35effff640 1 -- 172.21.15.78:0/3760299278 <== mon.0 v2:172.21.15.78:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 2}]=0 v0) ==== 74+0+13 (secure 0 0 0) 0x7f35f004b030 con 0x7f35f812bf30 2024-09-21T09:22:35.895 INFO:teuthology.orchestra.run.smithi078.stdout:115964116994 2024-09-21T09:22:35.898 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.895+0000 7f35fd9fe640 1 -- 172.21.15.78:0/3760299278 >> v1:172.21.15.78:6816/2186157269 conn(0x7f35d80514b0 legacy=0x7f35d8053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:35.898 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.895+0000 7f35fd9fe640 1 -- 172.21.15.78:0/3760299278 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f35f812bf30 msgr2=0x7f35f8070c60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:35.898 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.895+0000 7f35fd9fe640 1 --2- 172.21.15.78:0/3760299278 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f35f812bf30 0x7f35f8070c60 secure :-1 s=READY pgs=117 cs=0 l=1 rev1=1 crypto rx=0x7f35f0009be0 tx=0x7f35f0003c00 comp rx=0 tx=0).stop 2024-09-21T09:22:35.898 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.895+0000 7f35fd9fe640 1 -- 172.21.15.78:0/3760299278 shutdown_connections 2024-09-21T09:22:35.898 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.895+0000 7f35fd9fe640 1 --2- 172.21.15.78:0/3760299278 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f35f812bf30 0x7f35f8070c60 unknown :-1 s=CLOSED pgs=117 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:35.898 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.895+0000 7f35fd9fe640 1 --2- 172.21.15.78:0/3760299278 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f35f8093140 0x7f35f81c31c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:35.898 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.895+0000 7f35fd9fe640 1 --2- 172.21.15.78:0/3760299278 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f35f8092820 0x7f35f81c2c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:35.898 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.895+0000 7f35fd9fe640 1 -- 172.21.15.78:0/3760299278 >> 172.21.15.78:0/3760299278 conn(0x7f35f809e490 msgr2=0x7f35f8125340 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:35.898 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.895+0000 7f35fd9fe640 1 -- 172.21.15.78:0/3760299278 shutdown_connections 2024-09-21T09:22:35.898 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:35.895+0000 7f35fd9fe640 1 -- 172.21.15.78:0/3760299278 wait complete. 2024-09-21T09:22:35.903 INFO:tasks.ceph.ceph_manager.ceph:need seq 120259084291 got 1 for osd.3 2024-09-21T09:22:35.908 INFO:tasks.ceph.ceph_manager.ceph:need seq 115964116995 got 115964116994 for osd.2 2024-09-21T09:22:36.813 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.0 2024-09-21T09:22:36.827 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.1 2024-09-21T09:22:36.904 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.3 2024-09-21T09:22:36.909 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.2 2024-09-21T09:22:36.929 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.926+0000 7fbe2430f640 1 -- 172.21.15.78:0/188016701 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fbe1c12fb90 msgr2=0x7fbe1c131f80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:36.929 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.926+0000 7fbe2430f640 1 --2- 172.21.15.78:0/188016701 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fbe1c12fb90 0x7fbe1c131f80 secure :-1 s=READY pgs=119 cs=0 l=1 rev1=1 crypto rx=0x7fbe0c009830 tx=0x7fbe0c02a3f0 comp rx=0 tx=0).stop 2024-09-21T09:22:36.929 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.926+0000 7fbe2430f640 1 -- 172.21.15.78:0/188016701 shutdown_connections 2024-09-21T09:22:36.929 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.926+0000 7fbe2430f640 1 --2- 172.21.15.78:0/188016701 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fbe1c12fb90 0x7fbe1c131f80 unknown :-1 s=CLOSED pgs=119 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:36.929 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.926+0000 7fbe2430f640 1 --2- 172.21.15.78:0/188016701 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fbe1c1324c0 0x7fbe1c1348b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:36.929 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.926+0000 7fbe2430f640 1 -- 172.21.15.78:0/188016701 >> 172.21.15.78:0/188016701 conn(0x7fbe1c09d370 msgr2=0x7fbe1c09d780 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:36.930 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.926+0000 7fbe2430f640 1 -- 172.21.15.78:0/188016701 shutdown_connections 2024-09-21T09:22:36.930 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.926+0000 7fbe2430f640 1 -- 172.21.15.78:0/188016701 wait complete. 2024-09-21T09:22:36.932 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.927+0000 7fbe2430f640 1 Processor -- start 2024-09-21T09:22:36.932 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.927+0000 7fbe2430f640 1 -- start start 2024-09-21T09:22:36.932 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.928+0000 7fbe2430f640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fbe1c129510 0x7fbe1c12c2e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:36.932 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.928+0000 7fbe2430f640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fbe1c12fb90 0x7fbe1c12c820 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:36.932 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.928+0000 7fbe2430f640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fbe1c1324c0 0x7fbe1c12cd60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:36.933 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.928+0000 7fbe2430f640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7fbe1c135780 con 0x7fbe1c1324c0 2024-09-21T09:22:36.933 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.928+0000 7fbe2430f640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7fbe1c135600 con 0x7fbe1c12fb90 2024-09-21T09:22:36.933 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.928+0000 7fbe2430f640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7fbe1c135900 con 0x7fbe1c129510 2024-09-21T09:22:36.933 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.928+0000 7fbe228ac640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fbe1c1324c0 0x7fbe1c12cd60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:36.933 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.928+0000 7fbe228ac640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fbe1c1324c0 0x7fbe1c12cd60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3300/0 says I am v2:172.21.15.78:44550/0 (socket says 172.21.15.78:44550) 2024-09-21T09:22:36.933 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.928+0000 7fbe228ac640 1 -- 172.21.15.78:0/2441053958 learned_addr learned my addr 172.21.15.78:0/2441053958 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:36.933 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.928+0000 7fbe218aa640 1 --2- 172.21.15.78:0/2441053958 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fbe1c12fb90 0x7fbe1c12c820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:36.933 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.928+0000 7fbe220ab640 1 --2- 172.21.15.78:0/2441053958 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fbe1c129510 0x7fbe1c12c2e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:36.933 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.928+0000 7fbe228ac640 1 -- 172.21.15.78:0/2441053958 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fbe1c129510 msgr2=0x7fbe1c12c2e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:36.933 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.929+0000 7fbe228ac640 1 --2- 172.21.15.78:0/2441053958 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fbe1c129510 0x7fbe1c12c2e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:36.933 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.929+0000 7fbe228ac640 1 -- 172.21.15.78:0/2441053958 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fbe1c12fb90 msgr2=0x7fbe1c12c820 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:36.933 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.929+0000 7fbe228ac640 1 --2- 172.21.15.78:0/2441053958 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fbe1c12fb90 0x7fbe1c12c820 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:36.934 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.929+0000 7fbe228ac640 1 -- 172.21.15.78:0/2441053958 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fbe1c123f50 con 0x7fbe1c1324c0 2024-09-21T09:22:36.934 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.929+0000 7fbe218aa640 1 --2- 172.21.15.78:0/2441053958 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fbe1c12fb90 0x7fbe1c12c820 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:22:36.934 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.929+0000 7fbe220ab640 1 --2- 172.21.15.78:0/2441053958 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fbe1c129510 0x7fbe1c12c2e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-09-21T09:22:36.934 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.929+0000 7fbe228ac640 1 --2- 172.21.15.78:0/2441053958 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fbe1c1324c0 0x7fbe1c12cd60 secure :-1 s=READY pgs=120 cs=0 l=1 rev1=1 crypto rx=0x7fbe1400efd0 tx=0x7fbe1400c6a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:36.934 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.929+0000 7fbe12ffd640 1 -- 172.21.15.78:0/2441053958 <== mon.0 v2:172.21.15.78:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbe14003b70 con 0x7fbe1c1324c0 2024-09-21T09:22:36.934 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.929+0000 7fbe12ffd640 1 -- 172.21.15.78:0/2441053958 <== mon.0 v2:172.21.15.78:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fbe14003e00 con 0x7fbe1c1324c0 2024-09-21T09:22:36.934 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.929+0000 7fbe12ffd640 1 -- 172.21.15.78:0/2441053958 <== mon.0 v2:172.21.15.78:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbe14004280 con 0x7fbe1c1324c0 2024-09-21T09:22:36.934 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.929+0000 7fbe2430f640 1 -- 172.21.15.78:0/2441053958 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fbe1c126f70 con 0x7fbe1c1324c0 2024-09-21T09:22:36.934 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.929+0000 7fbe2430f640 1 -- 172.21.15.78:0/2441053958 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fbe1c125210 con 0x7fbe1c1324c0 2024-09-21T09:22:36.935 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.930+0000 7fbe12ffd640 1 -- 172.21.15.78:0/2441053958 <== mon.0 v2:172.21.15.78:3300/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7fbe14004420 con 0x7fbe1c1324c0 2024-09-21T09:22:36.935 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.931+0000 7fbe2430f640 1 -- 172.21.15.78:0/2441053958 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fbe1c127ab0 con 0x7fbe1c1324c0 2024-09-21T09:22:36.935 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.931+0000 7fbe12ffd640 1 -- 172.21.15.78:0/2441053958 <== mon.0 v2:172.21.15.78:3300/0 5 ==== osd_map(32..32 src has 1..32) ==== 3027+0+0 (secure 0 0 0) 0x7fbe1406a010 con 0x7fbe1c1324c0 2024-09-21T09:22:36.940 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.938+0000 7fbe12ffd640 1 -- 172.21.15.78:0/2441053958 <== mon.0 v2:172.21.15.78:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7fbe1c127ab0 con 0x7fbe1c1324c0 2024-09-21T09:22:36.977 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.974+0000 7fe3766c9640 1 -- 172.21.15.78:0/2786382968 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fe370129e30 msgr2=0x7fe37012c820 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:36.977 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.974+0000 7fe3766c9640 1 --2- 172.21.15.78:0/2786382968 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fe370129e30 0x7fe37012c820 secure :-1 s=READY pgs=50 cs=0 l=1 rev1=1 crypto rx=0x7fe364009830 tx=0x7fe36402a3f0 comp rx=0 tx=0).stop 2024-09-21T09:22:36.977 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.974+0000 7fe3766c9640 1 -- 172.21.15.78:0/2786382968 shutdown_connections 2024-09-21T09:22:36.977 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.974+0000 7fe3766c9640 1 --2- 172.21.15.78:0/2786382968 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fe370129e30 0x7fe37012c820 unknown :-1 s=CLOSED pgs=50 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:36.977 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.974+0000 7fe3766c9640 1 --2- 172.21.15.78:0/2786382968 >> v2:172.21.15.78:3300/0 conn(0x7fe370129510 0x7fe3701298f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:36.977 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.974+0000 7fe3766c9640 1 -- 172.21.15.78:0/2786382968 >> 172.21.15.78:0/2786382968 conn(0x7fe37009d370 msgr2=0x7fe37009d780 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:36.977 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.974+0000 7fe3766c9640 1 -- 172.21.15.78:0/2786382968 shutdown_connections 2024-09-21T09:22:36.978 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.974+0000 7fe3766c9640 1 -- 172.21.15.78:0/2786382968 wait complete. 2024-09-21T09:22:36.979 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.975+0000 7fe3766c9640 1 Processor -- start 2024-09-21T09:22:36.979 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.976+0000 7fe3766c9640 1 -- start start 2024-09-21T09:22:36.980 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.976+0000 7fe3766c9640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fe370129510 0x7fe3701222d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:36.980 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.976+0000 7fe3766c9640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fe370129e30 0x7fe370122810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:36.980 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.976+0000 7fe3766c9640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fe37012cd60 0x7fe37006f3e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:36.980 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.976+0000 7fe3766c9640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7fe370135740 con 0x7fe370129e30 2024-09-21T09:22:36.980 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.976+0000 7fe3766c9640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7fe3701355c0 con 0x7fe37012cd60 2024-09-21T09:22:36.980 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.976+0000 7fe3766c9640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7fe3701358c0 con 0x7fe370129510 2024-09-21T09:22:36.980 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.976+0000 7fe374c66640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fe37012cd60 0x7fe37006f3e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:36.980 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.976+0000 7fe374c66640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fe37012cd60 0x7fe37006f3e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3301/0 says I am v2:172.21.15.78:37500/0 (socket says 172.21.15.78:37500) 2024-09-21T09:22:36.980 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.976+0000 7fe374c66640 1 -- 172.21.15.78:0/2587090322 learned_addr learned my addr 172.21.15.78:0/2587090322 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:36.980 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.976+0000 7fe36f7fe640 1 --2- 172.21.15.78:0/2587090322 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fe370129e30 0x7fe370122810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:36.980 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.976+0000 7fe36ffff640 1 --2- 172.21.15.78:0/2587090322 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fe370129510 0x7fe3701222d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:36.980 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.976+0000 7fe374c66640 1 -- 172.21.15.78:0/2587090322 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fe370129510 msgr2=0x7fe3701222d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:36.980 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.976+0000 7fe374c66640 1 --2- 172.21.15.78:0/2587090322 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fe370129510 0x7fe3701222d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:36.981 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.976+0000 7fe374c66640 1 -- 172.21.15.78:0/2587090322 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fe370129e30 msgr2=0x7fe370122810 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:36.981 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.976+0000 7fe374c66640 1 --2- 172.21.15.78:0/2587090322 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fe370129e30 0x7fe370122810 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:36.981 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.976+0000 7fe374c66640 1 -- 172.21.15.78:0/2587090322 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fe370115cd0 con 0x7fe37012cd60 2024-09-21T09:22:36.981 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.976+0000 7fe36ffff640 1 --2- 172.21.15.78:0/2587090322 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fe370129510 0x7fe3701222d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-09-21T09:22:36.981 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.977+0000 7fe374c66640 1 --2- 172.21.15.78:0/2587090322 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fe37012cd60 0x7fe37006f3e0 secure :-1 s=READY pgs=51 cs=0 l=1 rev1=1 crypto rx=0x7fe36800cbf0 tx=0x7fe36800a060 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:36.981 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.977+0000 7fe36cff9640 1 -- 172.21.15.78:0/2587090322 <== mon.1 v2:172.21.15.78:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe368016020 con 0x7fe37012cd60 2024-09-21T09:22:36.981 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.977+0000 7fe36cff9640 1 -- 172.21.15.78:0/2587090322 <== mon.1 v2:172.21.15.78:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fe36800ad90 con 0x7fe37012cd60 2024-09-21T09:22:36.981 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.977+0000 7fe3766c9640 1 -- 172.21.15.78:0/2587090322 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fe370122d50 con 0x7fe37012cd60 2024-09-21T09:22:36.981 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.977+0000 7fe36cff9640 1 -- 172.21.15.78:0/2587090322 <== mon.1 v2:172.21.15.78:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe368012680 con 0x7fe37012cd60 2024-09-21T09:22:36.982 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.977+0000 7fe3766c9640 1 -- 172.21.15.78:0/2587090322 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7fe37006f920 con 0x7fe37012cd60 2024-09-21T09:22:36.982 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.978+0000 7fe36cff9640 1 -- 172.21.15.78:0/2587090322 <== mon.1 v2:172.21.15.78:3301/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7fe368019020 con 0x7fe37012cd60 2024-09-21T09:22:36.982 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.978+0000 7fe3766c9640 1 -- 172.21.15.78:0/2587090322 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fe3701298f0 con 0x7fe37012cd60 2024-09-21T09:22:36.982 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.979+0000 7fe36cff9640 1 -- 172.21.15.78:0/2587090322 <== mon.1 v2:172.21.15.78:3301/0 5 ==== osd_map(32..32 src has 1..32) ==== 3027+0+0 (secure 0 0 0) 0x7fe3680695f0 con 0x7fe37012cd60 2024-09-21T09:22:36.984 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:36.982+0000 7fe36cff9640 1 -- 172.21.15.78:0/2587090322 <== mon.1 v2:172.21.15.78:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7fe368020020 con 0x7fe37012cd60 2024-09-21T09:22:37.014 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.012+0000 7ff464f0f640 1 -- 172.21.15.78:0/1822418747 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7ff460098d40 msgr2=0x7ff460099120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:37.015 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.012+0000 7ff464f0f640 1 --2- 172.21.15.78:0/1822418747 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7ff460098d40 0x7ff460099120 secure :-1 s=READY pgs=52 cs=0 l=1 rev1=1 crypto rx=0x7ff450009860 tx=0x7ff45002a3d0 comp rx=0 tx=0).stop 2024-09-21T09:22:37.015 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.012+0000 7ff464f0f640 1 -- 172.21.15.78:0/1822418747 shutdown_connections 2024-09-21T09:22:37.015 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.012+0000 7ff464f0f640 1 --2- 172.21.15.78:0/1822418747 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7ff46012f1c0 0x7ff46012f650 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:37.015 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.012+0000 7ff464f0f640 1 --2- 172.21.15.78:0/1822418747 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7ff460098d40 0x7ff460099120 unknown :-1 s=CLOSED pgs=52 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:37.015 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.012+0000 7ff464f0f640 1 -- 172.21.15.78:0/1822418747 >> 172.21.15.78:0/1822418747 conn(0x7ff4600a1790 msgr2=0x7ff4600a1ba0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:37.015 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.012+0000 7ff464f0f640 1 -- 172.21.15.78:0/1822418747 shutdown_connections 2024-09-21T09:22:37.015 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.012+0000 7ff464f0f640 1 -- 172.21.15.78:0/1822418747 wait complete. 2024-09-21T09:22:37.017 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.013+0000 7ff464f0f640 1 Processor -- start 2024-09-21T09:22:37.017 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.013+0000 7ff464f0f640 1 -- start start 2024-09-21T09:22:37.017 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.013+0000 7ff464f0f640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7ff460098d40 0x7ff4601c4fd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:37.017 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.013+0000 7ff464f0f640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7ff460099660 0x7ff4601c5510 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:37.017 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.013+0000 7ff464f0f640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7ff46012f1c0 0x7ff4600713d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:37.018 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.013+0000 7ff464f0f640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7ff460137910 con 0x7ff460099660 2024-09-21T09:22:37.018 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.013+0000 7ff464f0f640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7ff460137790 con 0x7ff46012f1c0 2024-09-21T09:22:37.018 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.013+0000 7ff464f0f640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7ff460137a90 con 0x7ff460098d40 2024-09-21T09:22:37.018 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.013+0000 7ff45e59c640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7ff460098d40 0x7ff4601c4fd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:37.018 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.013+0000 7ff45e59c640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7ff460098d40 0x7ff4601c4fd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3302/0 says I am v2:172.21.15.78:35738/0 (socket says 172.21.15.78:35738) 2024-09-21T09:22:37.018 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.013+0000 7ff45e59c640 1 -- 172.21.15.78:0/1553011081 learned_addr learned my addr 172.21.15.78:0/1553011081 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:37.018 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.013+0000 7ff45ed9d640 1 --2- 172.21.15.78:0/1553011081 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7ff46012f1c0 0x7ff4600713d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:37.018 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.014+0000 7ff45dd9b640 1 --2- 172.21.15.78:0/1553011081 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7ff460099660 0x7ff4601c5510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:37.018 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.014+0000 7ff45e59c640 1 -- 172.21.15.78:0/1553011081 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7ff46012f1c0 msgr2=0x7ff4600713d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:37.018 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.014+0000 7ff45e59c640 1 --2- 172.21.15.78:0/1553011081 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7ff46012f1c0 0x7ff4600713d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:37.018 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.014+0000 7ff45e59c640 1 -- 172.21.15.78:0/1553011081 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7ff460099660 msgr2=0x7ff4601c5510 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:37.018 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.014+0000 7ff45e59c640 1 --2- 172.21.15.78:0/1553011081 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7ff460099660 0x7ff4601c5510 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:37.018 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.014+0000 7ff45e59c640 1 -- 172.21.15.78:0/1553011081 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ff460073890 con 0x7ff460098d40 2024-09-21T09:22:37.019 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.014+0000 7ff45dd9b640 1 --2- 172.21.15.78:0/1553011081 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7ff460099660 0x7ff4601c5510 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:22:37.019 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.014+0000 7ff45e59c640 1 --2- 172.21.15.78:0/1553011081 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7ff460098d40 0x7ff4601c4fd0 secure :-1 s=READY pgs=50 cs=0 l=1 rev1=1 crypto rx=0x7ff450009780 tx=0x7ff450033ba0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:37.019 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.014+0000 7ff44effd640 1 -- 172.21.15.78:0/1553011081 <== mon.2 v2:172.21.15.78:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff450031b50 con 0x7ff460098d40 2024-09-21T09:22:37.019 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.014+0000 7ff44effd640 1 -- 172.21.15.78:0/1553011081 <== mon.2 v2:172.21.15.78:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7ff450031e00 con 0x7ff460098d40 2024-09-21T09:22:37.019 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.014+0000 7ff464f0f640 1 -- 172.21.15.78:0/1553011081 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7ff46012f650 con 0x7ff460098d40 2024-09-21T09:22:37.019 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.015+0000 7ff44effd640 1 -- 172.21.15.78:0/1553011081 <== mon.2 v2:172.21.15.78:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff450005280 con 0x7ff460098d40 2024-09-21T09:22:37.019 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.015+0000 7ff464f0f640 1 -- 172.21.15.78:0/1553011081 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7ff460073d70 con 0x7ff460098d40 2024-09-21T09:22:37.019 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.015+0000 7ff464f0f640 1 -- 172.21.15.78:0/1553011081 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7ff460099120 con 0x7ff460098d40 2024-09-21T09:22:37.019 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.016+0000 7ff44effd640 1 -- 172.21.15.78:0/1553011081 <== mon.2 v2:172.21.15.78:3302/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7ff450005420 con 0x7ff460098d40 2024-09-21T09:22:37.020 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.016+0000 7ff44effd640 1 -- 172.21.15.78:0/1553011081 <== mon.2 v2:172.21.15.78:3302/0 5 ==== osd_map(32..32 src has 1..32) ==== 3027+0+0 (secure 0 0 0) 0x7ff450089ef0 con 0x7ff460098d40 2024-09-21T09:22:37.025 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.023+0000 7ff44effd640 1 -- 172.21.15.78:0/1553011081 <== mon.2 v2:172.21.15.78:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7ff45003f030 con 0x7ff460098d40 2024-09-21T09:22:37.039 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.036+0000 7ff344997640 1 -- 172.21.15.78:0/847720579 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7ff340099660 msgr2=0x7ff34012eab0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:37.039 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.036+0000 7ff344997640 1 --2- 172.21.15.78:0/847720579 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7ff340099660 0x7ff34012eab0 secure :-1 s=READY pgs=123 cs=0 l=1 rev1=1 crypto rx=0x7ff334005fd0 tx=0x7ff33402a410 comp rx=0 tx=0).stop 2024-09-21T09:22:37.039 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.036+0000 7ff344997640 1 -- 172.21.15.78:0/847720579 shutdown_connections 2024-09-21T09:22:37.039 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.036+0000 7ff344997640 1 --2- 172.21.15.78:0/847720579 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7ff34012eff0 0x7ff34012f480 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:37.039 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.036+0000 7ff344997640 1 --2- 172.21.15.78:0/847720579 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7ff340099660 0x7ff34012eab0 unknown :-1 s=CLOSED pgs=123 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:37.039 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.036+0000 7ff344997640 1 --2- 172.21.15.78:0/847720579 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7ff340098d40 0x7ff340099120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:37.039 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.036+0000 7ff344997640 1 -- 172.21.15.78:0/847720579 >> 172.21.15.78:0/847720579 conn(0x7ff3400a1790 msgr2=0x7ff3400a1ba0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:37.039 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.036+0000 7ff344997640 1 -- 172.21.15.78:0/847720579 shutdown_connections 2024-09-21T09:22:37.039 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.036+0000 7ff344997640 1 -- 172.21.15.78:0/847720579 wait complete. 2024-09-21T09:22:37.042 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.037+0000 7ff344997640 1 Processor -- start 2024-09-21T09:22:37.042 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.037+0000 7ff344997640 1 -- start start 2024-09-21T09:22:37.042 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.038+0000 7ff344997640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7ff340098d40 0x7ff3401c4c70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:37.042 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.038+0000 7ff344997640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7ff340099660 0x7ff3401c51b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:37.042 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.038+0000 7ff344997640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7ff34012eff0 0x7ff340132aa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:37.042 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.038+0000 7ff344997640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7ff340136570 con 0x7ff340099660 2024-09-21T09:22:37.042 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.038+0000 7ff344997640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7ff3401363f0 con 0x7ff340098d40 2024-09-21T09:22:37.042 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.038+0000 7ff344997640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7ff3401366f0 con 0x7ff34012eff0 2024-09-21T09:22:37.042 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.038+0000 7ff33e59c640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7ff340098d40 0x7ff3401c4c70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:37.042 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.038+0000 7ff33ed9d640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7ff34012eff0 0x7ff340132aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:37.042 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.038+0000 7ff33e59c640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7ff340098d40 0x7ff3401c4c70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3301/0 says I am v2:172.21.15.78:37534/0 (socket says 172.21.15.78:37534) 2024-09-21T09:22:37.043 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.038+0000 7ff33ed9d640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7ff34012eff0 0x7ff340132aa0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3302/0 says I am v2:172.21.15.78:35768/0 (socket says 172.21.15.78:35768) 2024-09-21T09:22:37.043 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.038+0000 7ff33e59c640 1 -- 172.21.15.78:0/963422397 learned_addr learned my addr 172.21.15.78:0/963422397 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:37.043 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.038+0000 7ff33dd9b640 1 --2- 172.21.15.78:0/963422397 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7ff340099660 0x7ff3401c51b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:37.043 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.038+0000 7ff33e59c640 1 -- 172.21.15.78:0/963422397 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7ff34012eff0 msgr2=0x7ff340132aa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:37.043 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.038+0000 7ff33e59c640 1 --2- 172.21.15.78:0/963422397 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7ff34012eff0 0x7ff340132aa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:37.043 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.038+0000 7ff33e59c640 1 -- 172.21.15.78:0/963422397 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7ff340099660 msgr2=0x7ff3401c51b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:37.043 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.038+0000 7ff33e59c640 1 --2- 172.21.15.78:0/963422397 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7ff340099660 0x7ff3401c51b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:37.043 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.038+0000 7ff33e59c640 1 -- 172.21.15.78:0/963422397 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ff340134ac0 con 0x7ff340098d40 2024-09-21T09:22:37.043 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.038+0000 7ff33dd9b640 1 --2- 172.21.15.78:0/963422397 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7ff340099660 0x7ff3401c51b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:22:37.043 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.038+0000 7ff33e59c640 1 --2- 172.21.15.78:0/963422397 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7ff340098d40 0x7ff3401c4c70 secure :-1 s=READY pgs=53 cs=0 l=1 rev1=1 crypto rx=0x7ff33000e7c0 tx=0x7ff33000ec80 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:37.043 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.039+0000 7ff32effd640 1 -- 172.21.15.78:0/963422397 <== mon.1 v2:172.21.15.78:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff33000ccc0 con 0x7ff340098d40 2024-09-21T09:22:37.043 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.039+0000 7ff344997640 1 -- 172.21.15.78:0/963422397 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7ff340134cc0 con 0x7ff340098d40 2024-09-21T09:22:37.043 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.039+0000 7ff344997640 1 -- 172.21.15.78:0/963422397 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7ff3401c56f0 con 0x7ff340098d40 2024-09-21T09:22:37.043 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.039+0000 7ff32effd640 1 -- 172.21.15.78:0/963422397 <== mon.1 v2:172.21.15.78:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7ff33000ce80 con 0x7ff340098d40 2024-09-21T09:22:37.044 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.039+0000 7ff32effd640 1 -- 172.21.15.78:0/963422397 <== mon.1 v2:172.21.15.78:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff330005280 con 0x7ff340098d40 2024-09-21T09:22:37.044 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.040+0000 7ff344997640 1 -- 172.21.15.78:0/963422397 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7ff324005290 con 0x7ff340098d40 2024-09-21T09:22:37.044 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.040+0000 7ff32effd640 1 -- 172.21.15.78:0/963422397 <== mon.1 v2:172.21.15.78:3301/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7ff330005420 con 0x7ff340098d40 2024-09-21T09:22:37.044 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.040+0000 7ff32effd640 1 -- 172.21.15.78:0/963422397 <== mon.1 v2:172.21.15.78:3301/0 5 ==== osd_map(32..32 src has 1..32) ==== 3027+0+0 (secure 0 0 0) 0x7ff330069860 con 0x7ff340098d40 2024-09-21T09:22:37.046 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.044+0000 7ff32effd640 1 -- 172.21.15.78:0/963422397 <== mon.1 v2:172.21.15.78:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7ff3300688b0 con 0x7ff340098d40 2024-09-21T09:22:37.185 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.183+0000 7ff464f0f640 1 -- 172.21.15.78:0/1553011081 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 3} v 0) -- 0x7ff4600698c0 con 0x7ff460098d40 2024-09-21T09:22:37.186 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.183+0000 7ff44effd640 1 -- 172.21.15.78:0/1553011081 <== mon.2 v2:172.21.15.78:3302/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 3}]=0 v0) ==== 74+0+13 (secure 0 0 0) 0x7ff450088fa0 con 0x7ff460098d40 2024-09-21T09:22:37.186 INFO:teuthology.orchestra.run.smithi078.stdout:120259084291 2024-09-21T09:22:37.188 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.185+0000 7ff464f0f640 1 -- 172.21.15.78:0/1553011081 >> v1:172.21.15.78:6816/2186157269 conn(0x7ff4400514b0 legacy=0x7ff440053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:37.188 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.185+0000 7ff464f0f640 1 -- 172.21.15.78:0/1553011081 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7ff460098d40 msgr2=0x7ff4601c4fd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:37.188 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.185+0000 7ff464f0f640 1 --2- 172.21.15.78:0/1553011081 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7ff460098d40 0x7ff4601c4fd0 secure :-1 s=READY pgs=50 cs=0 l=1 rev1=1 crypto rx=0x7ff450009780 tx=0x7ff450033ba0 comp rx=0 tx=0).stop 2024-09-21T09:22:37.188 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.185+0000 7ff464f0f640 1 -- 172.21.15.78:0/1553011081 shutdown_connections 2024-09-21T09:22:37.188 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.185+0000 7ff464f0f640 1 --2- 172.21.15.78:0/1553011081 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7ff46012f1c0 0x7ff4600713d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:37.188 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.185+0000 7ff464f0f640 1 --2- 172.21.15.78:0/1553011081 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7ff460099660 0x7ff4601c5510 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:37.188 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.185+0000 7ff464f0f640 1 --2- 172.21.15.78:0/1553011081 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7ff460098d40 0x7ff4601c4fd0 unknown :-1 s=CLOSED pgs=50 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:37.188 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.185+0000 7ff464f0f640 1 -- 172.21.15.78:0/1553011081 >> 172.21.15.78:0/1553011081 conn(0x7ff4600a1790 msgr2=0x7ff46009cf40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:37.188 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.185+0000 7ff464f0f640 1 -- 172.21.15.78:0/1553011081 shutdown_connections 2024-09-21T09:22:37.188 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.185+0000 7ff464f0f640 1 -- 172.21.15.78:0/1553011081 wait complete. 2024-09-21T09:22:37.199 INFO:tasks.ceph.ceph_manager.ceph:need seq 120259084291 got 120259084291 for osd.3 2024-09-21T09:22:37.199 DEBUG:teuthology.parallel:result is None 2024-09-21T09:22:37.203 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.201+0000 7ff344997640 1 -- 172.21.15.78:0/963422397 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 2} v 0) -- 0x7ff324005580 con 0x7ff340098d40 2024-09-21T09:22:37.203 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.201+0000 7ff32effd640 1 -- 172.21.15.78:0/963422397 <== mon.1 v2:172.21.15.78:3301/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 2}]=0 v0) ==== 74+0+13 (secure 0 0 0) 0x7ff3300683f0 con 0x7ff340098d40 2024-09-21T09:22:37.203 INFO:teuthology.orchestra.run.smithi078.stdout:115964116995 2024-09-21T09:22:37.205 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.203+0000 7ff344997640 1 -- 172.21.15.78:0/963422397 >> v1:172.21.15.78:6816/2186157269 conn(0x7ff3200514b0 legacy=0x7ff320053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:37.205 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.203+0000 7ff344997640 1 -- 172.21.15.78:0/963422397 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7ff340098d40 msgr2=0x7ff3401c4c70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:37.205 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.203+0000 7ff344997640 1 --2- 172.21.15.78:0/963422397 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7ff340098d40 0x7ff3401c4c70 secure :-1 s=READY pgs=53 cs=0 l=1 rev1=1 crypto rx=0x7ff33000e7c0 tx=0x7ff33000ec80 comp rx=0 tx=0).stop 2024-09-21T09:22:37.205 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.203+0000 7ff344997640 1 -- 172.21.15.78:0/963422397 shutdown_connections 2024-09-21T09:22:37.206 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.203+0000 7ff344997640 1 --2- 172.21.15.78:0/963422397 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7ff34012eff0 0x7ff340132aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:37.206 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.203+0000 7ff344997640 1 --2- 172.21.15.78:0/963422397 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7ff340099660 0x7ff3401c51b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:37.206 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.203+0000 7ff344997640 1 --2- 172.21.15.78:0/963422397 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7ff340098d40 0x7ff3401c4c70 unknown :-1 s=CLOSED pgs=53 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:37.206 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.203+0000 7ff344997640 1 -- 172.21.15.78:0/963422397 >> 172.21.15.78:0/963422397 conn(0x7ff3400a1790 msgr2=0x7ff340128440 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:37.206 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.203+0000 7ff344997640 1 -- 172.21.15.78:0/963422397 shutdown_connections 2024-09-21T09:22:37.206 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.203+0000 7ff344997640 1 -- 172.21.15.78:0/963422397 wait complete. 2024-09-21T09:22:37.216 INFO:tasks.ceph.ceph_manager.ceph:need seq 115964116995 got 115964116995 for osd.2 2024-09-21T09:22:37.216 DEBUG:teuthology.parallel:result is None 2024-09-21T09:22:37.219 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.217+0000 7fbe2430f640 1 -- 172.21.15.78:0/2441053958 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 0} v 0) -- 0x7fbe1c1298f0 con 0x7fbe1c1324c0 2024-09-21T09:22:37.220 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.218+0000 7fbe12ffd640 1 -- 172.21.15.78:0/2441053958 <== mon.0 v2:172.21.15.78:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 0}]=0 v0) ==== 74+0+13 (secure 0 0 0) 0x7fbe14004710 con 0x7fbe1c1324c0 2024-09-21T09:22:37.220 INFO:teuthology.orchestra.run.smithi078.stdout:111669149699 2024-09-21T09:22:37.222 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.220+0000 7fbe2430f640 1 -- 172.21.15.78:0/2441053958 >> v1:172.21.15.78:6816/2186157269 conn(0x7fbe000514b0 legacy=0x7fbe00053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:37.222 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.220+0000 7fbe2430f640 1 -- 172.21.15.78:0/2441053958 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fbe1c1324c0 msgr2=0x7fbe1c12cd60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:37.223 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.220+0000 7fbe2430f640 1 --2- 172.21.15.78:0/2441053958 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fbe1c1324c0 0x7fbe1c12cd60 secure :-1 s=READY pgs=120 cs=0 l=1 rev1=1 crypto rx=0x7fbe1400efd0 tx=0x7fbe1400c6a0 comp rx=0 tx=0).stop 2024-09-21T09:22:37.223 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.220+0000 7fbe2430f640 1 -- 172.21.15.78:0/2441053958 shutdown_connections 2024-09-21T09:22:37.223 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.220+0000 7fbe2430f640 1 --2- 172.21.15.78:0/2441053958 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fbe1c1324c0 0x7fbe1c12cd60 unknown :-1 s=CLOSED pgs=120 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:37.223 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.220+0000 7fbe2430f640 1 --2- 172.21.15.78:0/2441053958 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fbe1c12fb90 0x7fbe1c12c820 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:37.223 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.220+0000 7fbe2430f640 1 --2- 172.21.15.78:0/2441053958 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fbe1c129510 0x7fbe1c12c2e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:37.223 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.220+0000 7fbe2430f640 1 -- 172.21.15.78:0/2441053958 >> 172.21.15.78:0/2441053958 conn(0x7fbe1c09d370 msgr2=0x7fbe1c127d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:37.223 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.220+0000 7fbe2430f640 1 -- 172.21.15.78:0/2441053958 shutdown_connections 2024-09-21T09:22:37.223 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.220+0000 7fbe2430f640 1 -- 172.21.15.78:0/2441053958 wait complete. 2024-09-21T09:22:37.233 INFO:tasks.ceph.ceph_manager.ceph:need seq 111669149699 got 111669149699 for osd.0 2024-09-21T09:22:37.233 DEBUG:teuthology.parallel:result is None 2024-09-21T09:22:37.250 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.248+0000 7fe3766c9640 1 -- 172.21.15.78:0/2587090322 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 1} v 0) -- 0x7fe3700698c0 con 0x7fe37012cd60 2024-09-21T09:22:37.250 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.248+0000 7fe36cff9640 1 -- 172.21.15.78:0/2587090322 <== mon.1 v2:172.21.15.78:3301/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 1}]=0 v0) ==== 74+0+13 (secure 0 0 0) 0x7fe3680686a0 con 0x7fe37012cd60 2024-09-21T09:22:37.250 INFO:teuthology.orchestra.run.smithi078.stdout:111669149699 2024-09-21T09:22:37.252 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.250+0000 7fe3766c9640 1 -- 172.21.15.78:0/2587090322 >> v1:172.21.15.78:6816/2186157269 conn(0x7fe35c0514b0 legacy=0x7fe35c053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:37.253 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.250+0000 7fe3766c9640 1 -- 172.21.15.78:0/2587090322 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fe37012cd60 msgr2=0x7fe37006f3e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:37.253 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.250+0000 7fe3766c9640 1 --2- 172.21.15.78:0/2587090322 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fe37012cd60 0x7fe37006f3e0 secure :-1 s=READY pgs=51 cs=0 l=1 rev1=1 crypto rx=0x7fe36800cbf0 tx=0x7fe36800a060 comp rx=0 tx=0).stop 2024-09-21T09:22:37.253 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.250+0000 7fe3766c9640 1 -- 172.21.15.78:0/2587090322 shutdown_connections 2024-09-21T09:22:37.253 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.250+0000 7fe3766c9640 1 --2- 172.21.15.78:0/2587090322 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fe37012cd60 0x7fe37006f3e0 unknown :-1 s=CLOSED pgs=51 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:37.253 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.250+0000 7fe3766c9640 1 --2- 172.21.15.78:0/2587090322 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fe370129e30 0x7fe370122810 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:37.253 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.250+0000 7fe3766c9640 1 --2- 172.21.15.78:0/2587090322 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fe370129510 0x7fe3701222d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:37.253 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.250+0000 7fe3766c9640 1 -- 172.21.15.78:0/2587090322 >> 172.21.15.78:0/2587090322 conn(0x7fe37009d370 msgr2=0x7fe370134b00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:37.253 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.250+0000 7fe3766c9640 1 -- 172.21.15.78:0/2587090322 shutdown_connections 2024-09-21T09:22:37.253 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.250+0000 7fe3766c9640 1 -- 172.21.15.78:0/2587090322 wait complete. 2024-09-21T09:22:37.263 INFO:tasks.ceph.ceph_manager.ceph:need seq 111669149699 got 111669149699 for osd.1 2024-09-21T09:22:37.263 DEBUG:teuthology.parallel:result is None 2024-09-21T09:22:37.263 INFO:tasks.ceph.ceph_manager.ceph:waiting for clean 2024-09-21T09:22:37.263 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-09-21T09:22:37.404 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.402+0000 7f37d58f2640 1 -- 172.21.15.78:0/2549830458 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f37d012fb30 msgr2=0x7f37d0131f20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:37.404 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.402+0000 7f37d58f2640 1 --2- 172.21.15.78:0/2549830458 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f37d012fb30 0x7f37d0131f20 secure :-1 s=READY pgs=54 cs=0 l=1 rev1=1 crypto rx=0x7f37c4009860 tx=0x7f37c402a3d0 comp rx=0 tx=0).stop 2024-09-21T09:22:37.404 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.402+0000 7f37d58f2640 1 -- 172.21.15.78:0/2549830458 shutdown_connections 2024-09-21T09:22:37.404 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.402+0000 7f37d58f2640 1 --2- 172.21.15.78:0/2549830458 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f37d012fb30 0x7f37d0131f20 unknown :-1 s=CLOSED pgs=54 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:37.405 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.402+0000 7f37d58f2640 1 --2- 172.21.15.78:0/2549830458 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f37d0132460 0x7f37d0134850 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:37.405 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.402+0000 7f37d58f2640 1 -- 172.21.15.78:0/2549830458 >> 172.21.15.78:0/2549830458 conn(0x7f37d009d480 msgr2=0x7f37d009d890 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:37.405 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.402+0000 7f37d58f2640 1 -- 172.21.15.78:0/2549830458 shutdown_connections 2024-09-21T09:22:37.405 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.402+0000 7f37d58f2640 1 -- 172.21.15.78:0/2549830458 wait complete. 2024-09-21T09:22:37.407 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.402+0000 7f37d58f2640 1 Processor -- start 2024-09-21T09:22:37.407 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.403+0000 7f37d58f2640 1 -- start start 2024-09-21T09:22:37.407 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.403+0000 7f37d58f2640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f37d01294b0 0x7f37d012a5a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:37.407 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.403+0000 7f37d58f2640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f37d012fb30 0x7f37d012aae0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:37.407 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.403+0000 7f37d58f2640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f37d0132460 0x7f37d012b020 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:37.407 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.403+0000 7f37d58f2640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7f37d0135820 con 0x7f37d0132460 2024-09-21T09:22:37.407 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.403+0000 7f37d58f2640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7f37d01356a0 con 0x7f37d012fb30 2024-09-21T09:22:37.407 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.403+0000 7f37d58f2640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7f37d01359a0 con 0x7f37d01294b0 2024-09-21T09:22:37.407 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.403+0000 7f37ce7fc640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f37d012fb30 0x7f37d012aae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:37.407 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.403+0000 7f37ceffd640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f37d01294b0 0x7f37d012a5a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:37.407 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.403+0000 7f37cf7fe640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f37d0132460 0x7f37d012b020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:37.407 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.403+0000 7f37ce7fc640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f37d012fb30 0x7f37d012aae0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3301/0 says I am v2:172.21.15.78:37554/0 (socket says 172.21.15.78:37554) 2024-09-21T09:22:37.408 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.403+0000 7f37ce7fc640 1 -- 172.21.15.78:0/1396710066 learned_addr learned my addr 172.21.15.78:0/1396710066 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:37.408 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.403+0000 7f37ce7fc640 1 -- 172.21.15.78:0/1396710066 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f37d01294b0 msgr2=0x7f37d012a5a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:37.408 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.403+0000 7f37ce7fc640 1 --2- 172.21.15.78:0/1396710066 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f37d01294b0 0x7f37d012a5a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:37.408 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.403+0000 7f37ce7fc640 1 -- 172.21.15.78:0/1396710066 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f37d0132460 msgr2=0x7f37d012b020 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:37.408 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.403+0000 7f37ce7fc640 1 --2- 172.21.15.78:0/1396710066 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f37d0132460 0x7f37d012b020 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:37.408 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.403+0000 7f37ce7fc640 1 -- 172.21.15.78:0/1396710066 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f37d012bab0 con 0x7f37d012fb30 2024-09-21T09:22:37.408 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.403+0000 7f37ceffd640 1 --2- 172.21.15.78:0/1396710066 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f37d01294b0 0x7f37d012a5a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:22:37.408 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.403+0000 7f37cf7fe640 1 --2- 172.21.15.78:0/1396710066 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f37d0132460 0x7f37d012b020 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:22:37.408 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.404+0000 7f37ce7fc640 1 --2- 172.21.15.78:0/1396710066 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f37d012fb30 0x7f37d012aae0 secure :-1 s=READY pgs=55 cs=0 l=1 rev1=1 crypto rx=0x7f37c4009980 tx=0x7f37c4032c10 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:37.408 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.404+0000 7f37b7fff640 1 -- 172.21.15.78:0/1396710066 <== mon.1 v2:172.21.15.78:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f37c4040070 con 0x7f37d012fb30 2024-09-21T09:22:37.408 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.404+0000 7f37b7fff640 1 -- 172.21.15.78:0/1396710066 <== mon.1 v2:172.21.15.78:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f37c4005310 con 0x7f37d012fb30 2024-09-21T09:22:37.408 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.404+0000 7f37b7fff640 1 -- 172.21.15.78:0/1396710066 <== mon.1 v2:172.21.15.78:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f37c4005c90 con 0x7f37d012fb30 2024-09-21T09:22:37.408 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.404+0000 7f37d58f2640 1 -- 172.21.15.78:0/1396710066 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f37d012b5f0 con 0x7f37d012fb30 2024-09-21T09:22:37.409 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.404+0000 7f37d58f2640 1 -- 172.21.15.78:0/1396710066 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7f37d01257a0 con 0x7f37d012fb30 2024-09-21T09:22:37.409 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.405+0000 7f37d58f2640 1 -- 172.21.15.78:0/1396710066 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f37d0125a00 con 0x7f37d012fb30 2024-09-21T09:22:37.409 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.405+0000 7f37b7fff640 1 -- 172.21.15.78:0/1396710066 <== mon.1 v2:172.21.15.78:3301/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f37c4039040 con 0x7f37d012fb30 2024-09-21T09:22:37.410 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.405+0000 7f37b7fff640 1 -- 172.21.15.78:0/1396710066 <== mon.1 v2:172.21.15.78:3301/0 5 ==== osd_map(32..32 src has 1..32) ==== 3027+0+0 (secure 0 0 0) 0x7f37c408a0c0 con 0x7f37d012fb30 2024-09-21T09:22:37.411 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.409+0000 7f37b7fff640 1 -- 172.21.15.78:0/1396710066 <== mon.1 v2:172.21.15.78:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f37d012bab0 con 0x7f37d012fb30 2024-09-21T09:22:37.561 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.558+0000 7f37d58f2640 1 -- 172.21.15.78:0/1396710066 --> v1:172.21.15.78:6816/2186157269 -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) -- 0x7f37d0129890 con 0x7f37bc0514b0 2024-09-21T09:22:37.561 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.559+0000 7f37b7fff640 1 -- 172.21.15.78:0/1396710066 <== mgr.14103 v1:172.21.15.78:6816/2186157269 1 ==== mgr_command_reply(tid 0: 0 dumped all) ==== 18+0+32026 (unknown 2965378022 0 3246280446) 0x7f37d0129890 con 0x7f37bc0514b0 2024-09-21T09:22:37.562 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:22:37.562 INFO:teuthology.orchestra.run.smithi078.stderr:dumped all 2024-09-21T09:22:37.564 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.561+0000 7f37d58f2640 1 -- 172.21.15.78:0/1396710066 >> v1:172.21.15.78:6816/2186157269 conn(0x7f37bc0514b0 legacy=0x7f37bc053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:37.564 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.561+0000 7f37d58f2640 1 -- 172.21.15.78:0/1396710066 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f37d012fb30 msgr2=0x7f37d012aae0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:37.564 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.561+0000 7f37d58f2640 1 --2- 172.21.15.78:0/1396710066 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f37d012fb30 0x7f37d012aae0 secure :-1 s=READY pgs=55 cs=0 l=1 rev1=1 crypto rx=0x7f37c4009980 tx=0x7f37c4032c10 comp rx=0 tx=0).stop 2024-09-21T09:22:37.564 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.561+0000 7f37d58f2640 1 -- 172.21.15.78:0/1396710066 shutdown_connections 2024-09-21T09:22:37.564 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.561+0000 7f37d58f2640 1 --2- 172.21.15.78:0/1396710066 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f37d0132460 0x7f37d012b020 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:37.564 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.561+0000 7f37d58f2640 1 --2- 172.21.15.78:0/1396710066 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f37d012fb30 0x7f37d012aae0 unknown :-1 s=CLOSED pgs=55 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:37.564 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.561+0000 7f37d58f2640 1 --2- 172.21.15.78:0/1396710066 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f37d01294b0 0x7f37d012a5a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:37.564 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.561+0000 7f37d58f2640 1 -- 172.21.15.78:0/1396710066 >> 172.21.15.78:0/1396710066 conn(0x7f37d009d480 msgr2=0x7f37d0127ec0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:37.564 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.561+0000 7f37d58f2640 1 -- 172.21.15.78:0/1396710066 shutdown_connections 2024-09-21T09:22:37.564 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.561+0000 7f37d58f2640 1 -- 172.21.15.78:0/1396710066 wait complete. 2024-09-21T09:22:37.574 INFO:teuthology.orchestra.run.smithi078.stdout:{"pg_ready":true,"pg_map":{"version":25,"stamp":"2024-09-21T09:22:36.402116+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":151,"num_write_kb":1896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":107,"ondisk_log_size":107,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":377487360,"kb_used":112236,"kb_used_data":2224,"kb_used_omap":17,"kb_used_meta":109934,"kb_avail":377375124,"statfs":{"total":386547056640,"available":386432126976,"internally_reserved":0,"allocated":2277376,"data_stored":1556658,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":17745,"internal_metadata":112573103},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.933957"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":49,"reported_epoch":32,"state":"active+clean","last_fresh":"2024-09-21T09:22:34.490846+0000","last_change":"2024-09-21T09:22:31.543815+0000","last_active":"2024-09-21T09:22:34.490846+0000","last_peered":"2024-09-21T09:22:34.490846+0000","last_clean":"2024-09-21T09:22:34.490846+0000","last_became_active":"2024-09-21T09:22:31.543451+0000","last_became_peered":"2024-09-21T09:22:31.543451+0000","last_unstale":"2024-09-21T09:22:34.490846+0000","last_undegraded":"2024-09-21T09:22:34.490846+0000","last_fullsized":"2024-09-21T09:22:34.490846+0000","mapping_epoch":28,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":29,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:25:48.514441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039605,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":40,"reported_epoch":32,"state":"active+clean","last_fresh":"2024-09-21T09:22:34.490419+0000","last_change":"2024-09-21T09:22:29.466574+0000","last_active":"2024-09-21T09:22:34.490419+0000","last_peered":"2024-09-21T09:22:34.490419+0000","last_clean":"2024-09-21T09:22:34.490419+0000","last_became_active":"2024-09-21T09:22:29.466289+0000","last_became_peered":"2024-09-21T09:22:29.466289+0000","last_unstale":"2024-09-21T09:22:34.490419+0000","last_undegraded":"2024-09-21T09:22:34.490419+0000","last_fullsized":"2024-09-21T09:22:34.490419+0000","mapping_epoch":26,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":27,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:11:52.267440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040851400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":49,"reported_epoch":32,"state":"active+clean","last_fresh":"2024-09-21T09:22:34.490724+0000","last_change":"2024-09-21T09:22:31.543852+0000","last_active":"2024-09-21T09:22:34.490724+0000","last_peered":"2024-09-21T09:22:34.490724+0000","last_clean":"2024-09-21T09:22:34.490724+0000","last_became_active":"2024-09-21T09:22:31.543538+0000","last_became_peered":"2024-09-21T09:22:31.543538+0000","last_unstale":"2024-09-21T09:22:34.490724+0000","last_undegraded":"2024-09-21T09:22:34.490724+0000","last_fullsized":"2024-09-21T09:22:34.490724+0000","mapping_epoch":28,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":29,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:04:07.910766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001005517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":40,"reported_epoch":32,"state":"active+clean","last_fresh":"2024-09-21T09:22:34.490274+0000","last_change":"2024-09-21T09:22:29.467296+0000","last_active":"2024-09-21T09:22:34.490274+0000","last_peered":"2024-09-21T09:22:34.490274+0000","last_clean":"2024-09-21T09:22:34.490274+0000","last_became_active":"2024-09-21T09:22:29.466923+0000","last_became_peered":"2024-09-21T09:22:29.466923+0000","last_unstale":"2024-09-21T09:22:34.490274+0000","last_undegraded":"2024-09-21T09:22:34.490274+0000","last_fullsized":"2024-09-21T09:22:34.490274+0000","mapping_epoch":26,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":27,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:59:07.385986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054560899999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"17'2","reported_seq":46,"reported_epoch":32,"state":"active+clean","last_fresh":"2024-09-21T09:22:34.490675+0000","last_change":"2024-09-21T09:22:31.472650+0000","last_active":"2024-09-21T09:22:34.490675+0000","last_peered":"2024-09-21T09:22:34.490675+0000","last_clean":"2024-09-21T09:22:34.490675+0000","last_became_active":"2024-09-21T09:22:31.472357+0000","last_became_peered":"2024-09-21T09:22:31.472357+0000","last_unstale":"2024-09-21T09:22:34.490675+0000","last_undegraded":"2024-09-21T09:22:34.490675+0000","last_fullsized":"2024-09-21T09:22:34.490675+0000","mapping_epoch":28,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":29,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:07:36.189205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011881540000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":37,"reported_epoch":32,"state":"active+clean","last_fresh":"2024-09-21T09:22:34.489357+0000","last_change":"2024-09-21T09:22:31.472256+0000","last_active":"2024-09-21T09:22:34.489357+0000","last_peered":"2024-09-21T09:22:34.489357+0000","last_clean":"2024-09-21T09:22:34.489357+0000","last_became_active":"2024-09-21T09:22:31.471983+0000","last_became_peered":"2024-09-21T09:22:31.471983+0000","last_unstale":"2024-09-21T09:22:34.489357+0000","last_undegraded":"2024-09-21T09:22:34.489357+0000","last_fullsized":"2024-09-21T09:22:34.489357+0000","mapping_epoch":28,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":29,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:01:34.840425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023051899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":44,"reported_epoch":32,"state":"active+clean","last_fresh":"2024-09-21T09:22:34.490328+0000","last_change":"2024-09-21T09:22:31.473209+0000","last_active":"2024-09-21T09:22:34.490328+0000","last_peered":"2024-09-21T09:22:34.490328+0000","last_clean":"2024-09-21T09:22:34.490328+0000","last_became_active":"2024-09-21T09:22:31.472948+0000","last_became_peered":"2024-09-21T09:22:31.472948+0000","last_unstale":"2024-09-21T09:22:34.490328+0000","last_undegraded":"2024-09-21T09:22:34.490328+0000","last_fullsized":"2024-09-21T09:22:34.490328+0000","mapping_epoch":28,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":29,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:44:51.405569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039429200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"15'1","reported_seq":42,"reported_epoch":32,"state":"active+clean","last_fresh":"2024-09-21T09:22:34.490515+0000","last_change":"2024-09-21T09:22:31.472253+0000","last_active":"2024-09-21T09:22:34.490515+0000","last_peered":"2024-09-21T09:22:34.490515+0000","last_clean":"2024-09-21T09:22:34.490515+0000","last_became_active":"2024-09-21T09:22:31.471933+0000","last_became_peered":"2024-09-21T09:22:31.471933+0000","last_unstale":"2024-09-21T09:22:34.490515+0000","last_undegraded":"2024-09-21T09:22:34.490515+0000","last_fullsized":"2024-09-21T09:22:34.490515+0000","mapping_epoch":28,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":29,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:57:08.422651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091827900000000004,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"20'104","reported_seq":190,"reported_epoch":32,"state":"active+clean","last_fresh":"2024-09-21T09:22:34.490159+0000","last_change":"2024-09-21T09:22:29.466956+0000","last_active":"2024-09-21T09:22:34.490159+0000","last_peered":"2024-09-21T09:22:34.490159+0000","last_clean":"2024-09-21T09:22:34.490159+0000","last_became_active":"2024-09-21T09:22:29.466649+0000","last_became_peered":"2024-09-21T09:22:29.466649+0000","last_unstale":"2024-09-21T09:22:34.490159+0000","last_undegraded":"2024-09-21T09:22:34.490159+0000","last_fullsized":"2024-09-21T09:22:34.490159+0000","mapping_epoch":26,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":27,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:51.278939+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:51.278939+0000","last_clean_scrub_stamp":"2024-09-21T09:20:51.278939+0000","objects_scrubbed":0,"log_size":104,"log_dups_size":0,"ondisk_log_size":104,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:02:40.936727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":149,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":4},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":149,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104,"ondisk_log_size":104,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":3,"up_from":28,"seq":120259084291,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27808,"kb_used_data":268,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":94344032,"statfs":{"total":96636764160,"available":96608288768,"internally_reserved":0,"allocated":274432,"data_stored":93990,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4435,"internal_metadata":28176045},"hb_peers":[0,1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":27,"seq":115964116995,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27796,"kb_used_data":264,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":94344044,"statfs":{"total":96636764160,"available":96608301056,"internally_reserved":0,"allocated":270336,"data_stored":93971,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4437,"internal_metadata":28176043},"hb_peers":[0,1,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":1,"up_from":26,"seq":111669149699,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":28320,"kb_used_data":848,"kb_used_omap":4,"kb_used_meta":27451,"kb_avail":94343520,"statfs":{"total":96636764160,"available":96607764480,"internally_reserved":0,"allocated":868352,"data_stored":684358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4436,"internal_metadata":28110508},"hb_peers":[0,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":0,"up_from":26,"seq":111669149699,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":28312,"kb_used_data":844,"kb_used_omap":4,"kb_used_meta":27451,"kb_avail":94343528,"statfs":{"total":96636764160,"available":96607772672,"internally_reserved":0,"allocated":864256,"data_stored":684339,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4437,"internal_metadata":28110507},"hb_peers":[1,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T09:22:37.575 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-09-21T09:22:37.678 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.672+0000 7f656c4a5640 1 -- 172.21.15.78:0/1202758925 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f6564131030 msgr2=0x7f6564133420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:37.678 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.672+0000 7f656c4a5640 1 --2- 172.21.15.78:0/1202758925 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f6564131030 0x7f6564133420 secure :-1 s=READY pgs=51 cs=0 l=1 rev1=1 crypto rx=0x7f655c005fd0 tx=0x7f655c02a980 comp rx=0 tx=0).stop 2024-09-21T09:22:37.679 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.672+0000 7f656c4a5640 1 -- 172.21.15.78:0/1202758925 shutdown_connections 2024-09-21T09:22:37.679 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.672+0000 7f656c4a5640 1 --2- 172.21.15.78:0/1202758925 >> v2:172.21.15.78:3300/0 conn(0x7f656412c330 0x7f6564130af0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:37.679 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.672+0000 7f656c4a5640 1 --2- 172.21.15.78:0/1202758925 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f6564131030 0x7f6564133420 unknown :-1 s=CLOSED pgs=51 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:37.679 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.672+0000 7f656c4a5640 1 -- 172.21.15.78:0/1202758925 >> 172.21.15.78:0/1202758925 conn(0x7f656409a2e0 msgr2=0x7f6564097ad0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:37.679 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.672+0000 7f656c4a5640 1 -- 172.21.15.78:0/1202758925 shutdown_connections 2024-09-21T09:22:37.679 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.672+0000 7f656c4a5640 1 -- 172.21.15.78:0/1202758925 wait complete. 2024-09-21T09:22:37.679 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.673+0000 7f656c4a5640 1 Processor -- start 2024-09-21T09:22:37.680 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.673+0000 7f656c4a5640 1 -- start start 2024-09-21T09:22:37.680 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.673+0000 7f656c4a5640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f65640991f0 0x7f656408c1f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:37.680 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.673+0000 7f656c4a5640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f656412c330 0x7f656408c730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:37.680 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.673+0000 7f656c4a5640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f6564131030 0x7f656408f570 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:37.680 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.673+0000 7f656c4a5640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7f65641346b0 con 0x7f6564131030 2024-09-21T09:22:37.680 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.673+0000 7f656c4a5640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7f6564134530 con 0x7f656412c330 2024-09-21T09:22:37.680 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.673+0000 7f656c4a5640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7f6564134830 con 0x7f65640991f0 2024-09-21T09:22:37.681 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.673+0000 7f656a241640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f65640991f0 0x7f656408c1f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:37.681 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.673+0000 7f6569a40640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f656412c330 0x7f656408c730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:37.681 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.673+0000 7f656aa42640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f6564131030 0x7f656408f570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:37.681 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.673+0000 7f6569a40640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f656412c330 0x7f656408c730 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3301/0 says I am v2:172.21.15.78:37558/0 (socket says 172.21.15.78:37558) 2024-09-21T09:22:37.681 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.673+0000 7f656a241640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f65640991f0 0x7f656408c1f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3302/0 says I am v2:172.21.15.78:35798/0 (socket says 172.21.15.78:35798) 2024-09-21T09:22:37.681 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.673+0000 7f6569a40640 1 -- 172.21.15.78:0/3203022518 learned_addr learned my addr 172.21.15.78:0/3203022518 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:37.681 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.673+0000 7f656aa42640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f6564131030 0x7f656408f570 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3300/0 says I am v2:172.21.15.78:44622/0 (socket says 172.21.15.78:44622) 2024-09-21T09:22:37.681 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.674+0000 7f6569a40640 1 -- 172.21.15.78:0/3203022518 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f65640991f0 msgr2=0x7f656408c1f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:37.682 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.674+0000 7f6569a40640 1 --2- 172.21.15.78:0/3203022518 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f65640991f0 0x7f656408c1f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:37.682 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.674+0000 7f6569a40640 1 -- 172.21.15.78:0/3203022518 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f6564131030 msgr2=0x7f656408f570 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:37.682 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.674+0000 7f6569a40640 1 --2- 172.21.15.78:0/3203022518 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f6564131030 0x7f656408f570 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:37.682 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.674+0000 7f6569a40640 1 -- 172.21.15.78:0/3203022518 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f656408dd70 con 0x7f656412c330 2024-09-21T09:22:37.682 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.674+0000 7f6569a40640 1 --2- 172.21.15.78:0/3203022518 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f656412c330 0x7f656408c730 secure :-1 s=READY pgs=56 cs=0 l=1 rev1=1 crypto rx=0x7f656000ea60 tx=0x7f656000ef20 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:37.682 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.674+0000 7f6556ffd640 1 -- 172.21.15.78:0/3203022518 <== mon.1 v2:172.21.15.78:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f656000cde0 con 0x7f656412c330 2024-09-21T09:22:37.682 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.674+0000 7f6556ffd640 1 -- 172.21.15.78:0/3203022518 <== mon.1 v2:172.21.15.78:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f6560004020 con 0x7f656412c330 2024-09-21T09:22:37.683 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.674+0000 7f656c4a5640 1 -- 172.21.15.78:0/3203022518 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f656408cc70 con 0x7f656412c330 2024-09-21T09:22:37.683 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.674+0000 7f6556ffd640 1 -- 172.21.15.78:0/3203022518 <== mon.1 v2:172.21.15.78:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6560004450 con 0x7f656412c330 2024-09-21T09:22:37.683 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.674+0000 7f656c4a5640 1 -- 172.21.15.78:0/3203022518 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7f656408e8e0 con 0x7f656412c330 2024-09-21T09:22:37.683 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.675+0000 7f656c4a5640 1 -- 172.21.15.78:0/3203022518 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f656412bc20 con 0x7f656412c330 2024-09-21T09:22:37.683 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.675+0000 7f6556ffd640 1 -- 172.21.15.78:0/3203022518 <== mon.1 v2:172.21.15.78:3301/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f65600035f0 con 0x7f656412c330 2024-09-21T09:22:37.683 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.676+0000 7f6556ffd640 1 -- 172.21.15.78:0/3203022518 <== mon.1 v2:172.21.15.78:3301/0 5 ==== osd_map(32..32 src has 1..32) ==== 3027+0+0 (secure 0 0 0) 0x7f65600699d0 con 0x7f656412c330 2024-09-21T09:22:37.684 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.679+0000 7f6556ffd640 1 -- 172.21.15.78:0/3203022518 <== mon.1 v2:172.21.15.78:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f6560068a80 con 0x7f656412c330 2024-09-21T09:22:37.827 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.825+0000 7f656c4a5640 1 -- 172.21.15.78:0/3203022518 --> v1:172.21.15.78:6816/2186157269 -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) -- 0x7f656408dfe0 con 0x7f65480514b0 2024-09-21T09:22:37.828 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.826+0000 7f6556ffd640 1 -- 172.21.15.78:0/3203022518 <== mgr.14103 v1:172.21.15.78:6816/2186157269 1 ==== mgr_command_reply(tid 0: 0 dumped all) ==== 18+0+32026 (unknown 2965378022 0 3246280446) 0x7f656408dfe0 con 0x7f65480514b0 2024-09-21T09:22:37.828 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:22:37.829 INFO:teuthology.orchestra.run.smithi078.stderr:dumped all 2024-09-21T09:22:37.830 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.827+0000 7f656c4a5640 1 -- 172.21.15.78:0/3203022518 >> v1:172.21.15.78:6816/2186157269 conn(0x7f65480514b0 legacy=0x7f6548053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:37.830 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.827+0000 7f656c4a5640 1 -- 172.21.15.78:0/3203022518 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f656412c330 msgr2=0x7f656408c730 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:37.830 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.827+0000 7f656c4a5640 1 --2- 172.21.15.78:0/3203022518 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f656412c330 0x7f656408c730 secure :-1 s=READY pgs=56 cs=0 l=1 rev1=1 crypto rx=0x7f656000ea60 tx=0x7f656000ef20 comp rx=0 tx=0).stop 2024-09-21T09:22:37.830 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.828+0000 7f656c4a5640 1 -- 172.21.15.78:0/3203022518 shutdown_connections 2024-09-21T09:22:37.830 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.828+0000 7f656c4a5640 1 --2- 172.21.15.78:0/3203022518 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f6564131030 0x7f656408f570 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:37.830 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.828+0000 7f656c4a5640 1 --2- 172.21.15.78:0/3203022518 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f656412c330 0x7f656408c730 unknown :-1 s=CLOSED pgs=56 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:37.831 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.828+0000 7f656c4a5640 1 --2- 172.21.15.78:0/3203022518 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f65640991f0 0x7f656408c1f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:37.831 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.828+0000 7f656c4a5640 1 -- 172.21.15.78:0/3203022518 >> 172.21.15.78:0/3203022518 conn(0x7f656409a2e0 msgr2=0x7f6564091c60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:37.831 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.828+0000 7f656c4a5640 1 -- 172.21.15.78:0/3203022518 shutdown_connections 2024-09-21T09:22:37.831 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.828+0000 7f656c4a5640 1 -- 172.21.15.78:0/3203022518 wait complete. 2024-09-21T09:22:37.841 INFO:teuthology.orchestra.run.smithi078.stdout:{"pg_ready":true,"pg_map":{"version":25,"stamp":"2024-09-21T09:22:36.402116+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":151,"num_write_kb":1896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":107,"ondisk_log_size":107,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":377487360,"kb_used":112236,"kb_used_data":2224,"kb_used_omap":17,"kb_used_meta":109934,"kb_avail":377375124,"statfs":{"total":386547056640,"available":386432126976,"internally_reserved":0,"allocated":2277376,"data_stored":1556658,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":17745,"internal_metadata":112573103},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.933957"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":49,"reported_epoch":32,"state":"active+clean","last_fresh":"2024-09-21T09:22:34.490846+0000","last_change":"2024-09-21T09:22:31.543815+0000","last_active":"2024-09-21T09:22:34.490846+0000","last_peered":"2024-09-21T09:22:34.490846+0000","last_clean":"2024-09-21T09:22:34.490846+0000","last_became_active":"2024-09-21T09:22:31.543451+0000","last_became_peered":"2024-09-21T09:22:31.543451+0000","last_unstale":"2024-09-21T09:22:34.490846+0000","last_undegraded":"2024-09-21T09:22:34.490846+0000","last_fullsized":"2024-09-21T09:22:34.490846+0000","mapping_epoch":28,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":29,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:25:48.514441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039605,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":40,"reported_epoch":32,"state":"active+clean","last_fresh":"2024-09-21T09:22:34.490419+0000","last_change":"2024-09-21T09:22:29.466574+0000","last_active":"2024-09-21T09:22:34.490419+0000","last_peered":"2024-09-21T09:22:34.490419+0000","last_clean":"2024-09-21T09:22:34.490419+0000","last_became_active":"2024-09-21T09:22:29.466289+0000","last_became_peered":"2024-09-21T09:22:29.466289+0000","last_unstale":"2024-09-21T09:22:34.490419+0000","last_undegraded":"2024-09-21T09:22:34.490419+0000","last_fullsized":"2024-09-21T09:22:34.490419+0000","mapping_epoch":26,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":27,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:11:52.267440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040851400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":49,"reported_epoch":32,"state":"active+clean","last_fresh":"2024-09-21T09:22:34.490724+0000","last_change":"2024-09-21T09:22:31.543852+0000","last_active":"2024-09-21T09:22:34.490724+0000","last_peered":"2024-09-21T09:22:34.490724+0000","last_clean":"2024-09-21T09:22:34.490724+0000","last_became_active":"2024-09-21T09:22:31.543538+0000","last_became_peered":"2024-09-21T09:22:31.543538+0000","last_unstale":"2024-09-21T09:22:34.490724+0000","last_undegraded":"2024-09-21T09:22:34.490724+0000","last_fullsized":"2024-09-21T09:22:34.490724+0000","mapping_epoch":28,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":29,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:04:07.910766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001005517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":40,"reported_epoch":32,"state":"active+clean","last_fresh":"2024-09-21T09:22:34.490274+0000","last_change":"2024-09-21T09:22:29.467296+0000","last_active":"2024-09-21T09:22:34.490274+0000","last_peered":"2024-09-21T09:22:34.490274+0000","last_clean":"2024-09-21T09:22:34.490274+0000","last_became_active":"2024-09-21T09:22:29.466923+0000","last_became_peered":"2024-09-21T09:22:29.466923+0000","last_unstale":"2024-09-21T09:22:34.490274+0000","last_undegraded":"2024-09-21T09:22:34.490274+0000","last_fullsized":"2024-09-21T09:22:34.490274+0000","mapping_epoch":26,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":27,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:59:07.385986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054560899999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"17'2","reported_seq":46,"reported_epoch":32,"state":"active+clean","last_fresh":"2024-09-21T09:22:34.490675+0000","last_change":"2024-09-21T09:22:31.472650+0000","last_active":"2024-09-21T09:22:34.490675+0000","last_peered":"2024-09-21T09:22:34.490675+0000","last_clean":"2024-09-21T09:22:34.490675+0000","last_became_active":"2024-09-21T09:22:31.472357+0000","last_became_peered":"2024-09-21T09:22:31.472357+0000","last_unstale":"2024-09-21T09:22:34.490675+0000","last_undegraded":"2024-09-21T09:22:34.490675+0000","last_fullsized":"2024-09-21T09:22:34.490675+0000","mapping_epoch":28,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":29,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:07:36.189205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011881540000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":37,"reported_epoch":32,"state":"active+clean","last_fresh":"2024-09-21T09:22:34.489357+0000","last_change":"2024-09-21T09:22:31.472256+0000","last_active":"2024-09-21T09:22:34.489357+0000","last_peered":"2024-09-21T09:22:34.489357+0000","last_clean":"2024-09-21T09:22:34.489357+0000","last_became_active":"2024-09-21T09:22:31.471983+0000","last_became_peered":"2024-09-21T09:22:31.471983+0000","last_unstale":"2024-09-21T09:22:34.489357+0000","last_undegraded":"2024-09-21T09:22:34.489357+0000","last_fullsized":"2024-09-21T09:22:34.489357+0000","mapping_epoch":28,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":29,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:01:34.840425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023051899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":44,"reported_epoch":32,"state":"active+clean","last_fresh":"2024-09-21T09:22:34.490328+0000","last_change":"2024-09-21T09:22:31.473209+0000","last_active":"2024-09-21T09:22:34.490328+0000","last_peered":"2024-09-21T09:22:34.490328+0000","last_clean":"2024-09-21T09:22:34.490328+0000","last_became_active":"2024-09-21T09:22:31.472948+0000","last_became_peered":"2024-09-21T09:22:31.472948+0000","last_unstale":"2024-09-21T09:22:34.490328+0000","last_undegraded":"2024-09-21T09:22:34.490328+0000","last_fullsized":"2024-09-21T09:22:34.490328+0000","mapping_epoch":28,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":29,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:44:51.405569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039429200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"15'1","reported_seq":42,"reported_epoch":32,"state":"active+clean","last_fresh":"2024-09-21T09:22:34.490515+0000","last_change":"2024-09-21T09:22:31.472253+0000","last_active":"2024-09-21T09:22:34.490515+0000","last_peered":"2024-09-21T09:22:34.490515+0000","last_clean":"2024-09-21T09:22:34.490515+0000","last_became_active":"2024-09-21T09:22:31.471933+0000","last_became_peered":"2024-09-21T09:22:31.471933+0000","last_unstale":"2024-09-21T09:22:34.490515+0000","last_undegraded":"2024-09-21T09:22:34.490515+0000","last_fullsized":"2024-09-21T09:22:34.490515+0000","mapping_epoch":28,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":29,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:57:08.422651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091827900000000004,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"20'104","reported_seq":190,"reported_epoch":32,"state":"active+clean","last_fresh":"2024-09-21T09:22:34.490159+0000","last_change":"2024-09-21T09:22:29.466956+0000","last_active":"2024-09-21T09:22:34.490159+0000","last_peered":"2024-09-21T09:22:34.490159+0000","last_clean":"2024-09-21T09:22:34.490159+0000","last_became_active":"2024-09-21T09:22:29.466649+0000","last_became_peered":"2024-09-21T09:22:29.466649+0000","last_unstale":"2024-09-21T09:22:34.490159+0000","last_undegraded":"2024-09-21T09:22:34.490159+0000","last_fullsized":"2024-09-21T09:22:34.490159+0000","mapping_epoch":26,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":27,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:51.278939+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:51.278939+0000","last_clean_scrub_stamp":"2024-09-21T09:20:51.278939+0000","objects_scrubbed":0,"log_size":104,"log_dups_size":0,"ondisk_log_size":104,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:02:40.936727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":149,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":4},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":149,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104,"ondisk_log_size":104,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":3,"up_from":28,"seq":120259084291,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27808,"kb_used_data":268,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":94344032,"statfs":{"total":96636764160,"available":96608288768,"internally_reserved":0,"allocated":274432,"data_stored":93990,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4435,"internal_metadata":28176045},"hb_peers":[0,1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":27,"seq":115964116995,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27796,"kb_used_data":264,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":94344044,"statfs":{"total":96636764160,"available":96608301056,"internally_reserved":0,"allocated":270336,"data_stored":93971,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4437,"internal_metadata":28176043},"hb_peers":[0,1,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":1,"up_from":26,"seq":111669149699,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":28320,"kb_used_data":848,"kb_used_omap":4,"kb_used_meta":27451,"kb_avail":94343520,"statfs":{"total":96636764160,"available":96607764480,"internally_reserved":0,"allocated":868352,"data_stored":684358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4436,"internal_metadata":28110508},"hb_peers":[0,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":0,"up_from":26,"seq":111669149699,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":28312,"kb_used_data":844,"kb_used_omap":4,"kb_used_meta":27451,"kb_avail":94343528,"statfs":{"total":96636764160,"available":96607772672,"internally_reserved":0,"allocated":864256,"data_stored":684339,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4437,"internal_metadata":28110507},"hb_peers":[1,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T09:22:37.842 INFO:tasks.ceph.ceph_manager.ceph:clean! 2024-09-21T09:22:37.842 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-21T09:22:37.842 INFO:tasks.ceph.ceph_manager.ceph:wait_until_healthy 2024-09-21T09:22:37.842 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph health --format=json 2024-09-21T09:22:37.940 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.937+0000 7f8710f49640 1 -- 172.21.15.78:0/3304618463 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f870c12f1c0 msgr2=0x7f870c12f610 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:37.940 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.937+0000 7f8710f49640 1 --2- 172.21.15.78:0/3304618463 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f870c12f1c0 0x7f870c12f610 secure :-1 s=READY pgs=57 cs=0 l=1 rev1=1 crypto rx=0x7f86fc009830 tx=0x7f86fc02a480 comp rx=0 tx=0).stop 2024-09-21T09:22:37.940 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.938+0000 7f8710f49640 1 -- 172.21.15.78:0/3304618463 shutdown_connections 2024-09-21T09:22:37.940 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.938+0000 7f8710f49640 1 --2- 172.21.15.78:0/3304618463 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f870c12f1c0 0x7f870c12f610 unknown :-1 s=CLOSED pgs=57 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:37.940 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.938+0000 7f8710f49640 1 --2- 172.21.15.78:0/3304618463 >> v2:172.21.15.78:3300/0 conn(0x7f870c099080 0x7f870c099460 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:37.940 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.938+0000 7f8710f49640 1 -- 172.21.15.78:0/3304618463 >> 172.21.15.78:0/3304618463 conn(0x7f870c0a1800 msgr2=0x7f870c0a1c10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:37.941 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.938+0000 7f8710f49640 1 -- 172.21.15.78:0/3304618463 shutdown_connections 2024-09-21T09:22:37.941 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.938+0000 7f8710f49640 1 -- 172.21.15.78:0/3304618463 wait complete. 2024-09-21T09:22:37.943 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.939+0000 7f8710f49640 1 Processor -- start 2024-09-21T09:22:37.943 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.939+0000 7f8710f49640 1 -- start start 2024-09-21T09:22:37.943 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.939+0000 7f8710f49640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f870c099080 0x7f870c071050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:37.943 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.939+0000 7f8710f49640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f870c0999a0 0x7f870c071590 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:37.943 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.939+0000 7f8710f49640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f870c12f1c0 0x7f870c071ad0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:37.943 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.939+0000 7f8710f49640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7f870c137660 con 0x7f870c099080 2024-09-21T09:22:37.943 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.939+0000 7f8710f49640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7f870c1374e0 con 0x7f870c0999a0 2024-09-21T09:22:37.943 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.939+0000 7f8710f49640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7f870c1377e0 con 0x7f870c12f1c0 2024-09-21T09:22:37.943 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.939+0000 7f870a59c640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f870c099080 0x7f870c071050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:37.944 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.939+0000 7f8709d9b640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f870c0999a0 0x7f870c071590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:37.944 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.939+0000 7f870a59c640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f870c099080 0x7f870c071050 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3300/0 says I am v2:172.21.15.78:44644/0 (socket says 172.21.15.78:44644) 2024-09-21T09:22:37.944 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.939+0000 7f870a59c640 1 -- 172.21.15.78:0/79790462 learned_addr learned my addr 172.21.15.78:0/79790462 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:37.944 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.939+0000 7f870ad9d640 1 --2- 172.21.15.78:0/79790462 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f870c12f1c0 0x7f870c071ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:37.944 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.939+0000 7f870a59c640 1 -- 172.21.15.78:0/79790462 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f870c12f1c0 msgr2=0x7f870c071ad0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:37.944 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.939+0000 7f870a59c640 1 --2- 172.21.15.78:0/79790462 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f870c12f1c0 0x7f870c071ad0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:37.944 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.939+0000 7f870a59c640 1 -- 172.21.15.78:0/79790462 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f870c0999a0 msgr2=0x7f870c071590 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:37.944 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.939+0000 7f870a59c640 1 --2- 172.21.15.78:0/79790462 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f870c0999a0 0x7f870c071590 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:37.944 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.939+0000 7f870a59c640 1 -- 172.21.15.78:0/79790462 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f870c073040 con 0x7f870c099080 2024-09-21T09:22:37.944 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.939+0000 7f8709d9b640 1 --2- 172.21.15.78:0/79790462 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f870c0999a0 0x7f870c071590 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:22:37.944 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.940+0000 7f870a59c640 1 --2- 172.21.15.78:0/79790462 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f870c099080 0x7f870c071050 secure :-1 s=READY pgs=127 cs=0 l=1 rev1=1 crypto rx=0x7f8704009d50 tx=0x7f870400c6a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:37.944 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.940+0000 7f8702ffd640 1 -- 172.21.15.78:0/79790462 <== mon.0 v2:172.21.15.78:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8704003b70 con 0x7f870c099080 2024-09-21T09:22:37.944 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.940+0000 7f8702ffd640 1 -- 172.21.15.78:0/79790462 <== mon.0 v2:172.21.15.78:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f8704003d10 con 0x7f870c099080 2024-09-21T09:22:37.944 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.940+0000 7f8702ffd640 1 -- 172.21.15.78:0/79790462 <== mon.0 v2:172.21.15.78:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8704004280 con 0x7f870c099080 2024-09-21T09:22:37.944 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.940+0000 7f8710f49640 1 -- 172.21.15.78:0/79790462 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f870c1294d0 con 0x7f870c099080 2024-09-21T09:22:37.945 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.940+0000 7f8710f49640 1 -- 172.21.15.78:0/79790462 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f870c136840 con 0x7f870c099080 2024-09-21T09:22:37.945 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.941+0000 7f8702ffd640 1 -- 172.21.15.78:0/79790462 <== mon.0 v2:172.21.15.78:3300/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f8704004420 con 0x7f870c099080 2024-09-21T09:22:37.945 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.941+0000 7f8710f49640 1 -- 172.21.15.78:0/79790462 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f870c136b00 con 0x7f870c099080 2024-09-21T09:22:37.945 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.941+0000 7f8702ffd640 1 -- 172.21.15.78:0/79790462 <== mon.0 v2:172.21.15.78:3300/0 5 ==== osd_map(32..32 src has 1..32) ==== 3027+0+0 (secure 0 0 0) 0x7f8704069ce0 con 0x7f870c099080 2024-09-21T09:22:37.947 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:37.945+0000 7f8702ffd640 1 -- 172.21.15.78:0/79790462 <== mon.0 v2:172.21.15.78:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f8704068d90 con 0x7f870c099080 2024-09-21T09:22:38.118 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.116+0000 7f8710f49640 1 -- 172.21.15.78:0/79790462 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "health", "format": "json"} v 0) -- 0x7f870c09d4f0 con 0x7f870c099080 2024-09-21T09:22:38.119 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.116+0000 7f8702ffd640 1 -- 172.21.15.78:0/79790462 <== mon.0 v2:172.21.15.78:3300/0 7 ==== mon_command_ack([{"prefix": "health", "format": "json"}]=0 v0) ==== 72+0+46 (secure 0 0 0) 0x7f8704004710 con 0x7f870c099080 2024-09-21T09:22:38.119 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:22:38.119 INFO:teuthology.orchestra.run.smithi078.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-21T09:22:38.120 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.118+0000 7f8710f49640 1 -- 172.21.15.78:0/79790462 >> v1:172.21.15.78:6816/2186157269 conn(0x7f86ec0514e0 legacy=0x7f86ec0539a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:38.121 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.118+0000 7f8710f49640 1 -- 172.21.15.78:0/79790462 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f870c099080 msgr2=0x7f870c071050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:38.121 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.118+0000 7f8710f49640 1 --2- 172.21.15.78:0/79790462 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f870c099080 0x7f870c071050 secure :-1 s=READY pgs=127 cs=0 l=1 rev1=1 crypto rx=0x7f8704009d50 tx=0x7f870400c6a0 comp rx=0 tx=0).stop 2024-09-21T09:22:38.121 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.118+0000 7f8710f49640 1 -- 172.21.15.78:0/79790462 shutdown_connections 2024-09-21T09:22:38.121 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.118+0000 7f8710f49640 1 --2- 172.21.15.78:0/79790462 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f870c12f1c0 0x7f870c071ad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:38.121 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.118+0000 7f8710f49640 1 --2- 172.21.15.78:0/79790462 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f870c0999a0 0x7f870c071590 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:38.121 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.118+0000 7f8710f49640 1 --2- 172.21.15.78:0/79790462 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f870c099080 0x7f870c071050 unknown :-1 s=CLOSED pgs=127 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:38.121 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.118+0000 7f8710f49640 1 -- 172.21.15.78:0/79790462 >> 172.21.15.78:0/79790462 conn(0x7f870c0a1800 msgr2=0x7f870c12bd30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:38.121 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.118+0000 7f8710f49640 1 -- 172.21.15.78:0/79790462 shutdown_connections 2024-09-21T09:22:38.121 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.118+0000 7f8710f49640 1 -- 172.21.15.78:0/79790462 wait complete. 2024-09-21T09:22:38.131 INFO:tasks.ceph.ceph_manager.ceph:wait_until_healthy done 2024-09-21T09:22:38.132 INFO:teuthology.run_tasks:Running task print... 2024-09-21T09:22:38.141 INFO:teuthology.task.print:**** done ceph.restart 2024-09-21T09:22:38.141 DEBUG:teuthology.run_tasks:Unwinding manager ceph.restart 2024-09-21T09:22:38.150 DEBUG:teuthology.run_tasks:Unwinding manager ceph.restart 2024-09-21T09:22:38.159 DEBUG:teuthology.run_tasks:Unwinding manager install.upgrade 2024-09-21T09:22:38.167 DEBUG:teuthology.run_tasks:Unwinding manager ceph 2024-09-21T09:22:38.176 INFO:tasks.ceph.ceph_manager.ceph:waiting for clean 2024-09-21T09:22:38.176 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-09-21T09:22:38.277 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.274+0000 7fb8ea1a2640 1 -- 172.21.15.78:0/3052932847 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fb8e4129210 msgr2=0x7fb8e41295f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:38.277 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.274+0000 7fb8ea1a2640 1 --2- 172.21.15.78:0/3052932847 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fb8e4129210 0x7fb8e41295f0 secure :-1 s=READY pgs=52 cs=0 l=1 rev1=1 crypto rx=0x7fb8d4009860 tx=0x7fb8d402a3d0 comp rx=0 tx=0).stop 2024-09-21T09:22:38.277 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.274+0000 7fb8ea1a2640 1 -- 172.21.15.78:0/3052932847 shutdown_connections 2024-09-21T09:22:38.277 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.274+0000 7fb8ea1a2640 1 --2- 172.21.15.78:0/3052932847 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fb8e4129b30 0x7fb8e412c520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:38.277 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.274+0000 7fb8ea1a2640 1 --2- 172.21.15.78:0/3052932847 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fb8e4129210 0x7fb8e41295f0 unknown :-1 s=CLOSED pgs=52 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:38.277 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.274+0000 7fb8ea1a2640 1 -- 172.21.15.78:0/3052932847 >> 172.21.15.78:0/3052932847 conn(0x7fb8e409d370 msgr2=0x7fb8e409d780 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:38.277 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.274+0000 7fb8ea1a2640 1 -- 172.21.15.78:0/3052932847 shutdown_connections 2024-09-21T09:22:38.277 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.274+0000 7fb8ea1a2640 1 -- 172.21.15.78:0/3052932847 wait complete. 2024-09-21T09:22:38.279 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.275+0000 7fb8ea1a2640 1 Processor -- start 2024-09-21T09:22:38.279 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.275+0000 7fb8ea1a2640 1 -- start start 2024-09-21T09:22:38.279 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.275+0000 7fb8ea1a2640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fb8e4129210 0x7fb8e4132f40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:38.279 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.275+0000 7fb8ea1a2640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fb8e4129b30 0x7fb8e4133480 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:38.280 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.275+0000 7fb8ea1a2640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fb8e412ca60 0x7fb8e41339c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:38.280 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.275+0000 7fb8ea1a2640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7fb8e4135600 con 0x7fb8e4129210 2024-09-21T09:22:38.280 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.275+0000 7fb8ea1a2640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7fb8e4135480 con 0x7fb8e4129b30 2024-09-21T09:22:38.280 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.275+0000 7fb8ea1a2640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7fb8e4135780 con 0x7fb8e412ca60 2024-09-21T09:22:38.280 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.275+0000 7fb8e91a0640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fb8e4129210 0x7fb8e4132f40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:38.280 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.275+0000 7fb8e99a1640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fb8e412ca60 0x7fb8e41339c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:38.280 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.276+0000 7fb8e91a0640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fb8e4129210 0x7fb8e4132f40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3300/0 says I am v2:172.21.15.78:44648/0 (socket says 172.21.15.78:44648) 2024-09-21T09:22:38.280 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.276+0000 7fb8e91a0640 1 -- 172.21.15.78:0/350373230 learned_addr learned my addr 172.21.15.78:0/350373230 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:38.280 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.276+0000 7fb8e899f640 1 --2- 172.21.15.78:0/350373230 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fb8e4129b30 0x7fb8e4133480 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:38.280 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.276+0000 7fb8e99a1640 1 -- 172.21.15.78:0/350373230 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fb8e4129b30 msgr2=0x7fb8e4133480 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:38.280 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.276+0000 7fb8e99a1640 1 --2- 172.21.15.78:0/350373230 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fb8e4129b30 0x7fb8e4133480 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:38.280 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.276+0000 7fb8e99a1640 1 -- 172.21.15.78:0/350373230 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fb8e4129210 msgr2=0x7fb8e4132f40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:38.280 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.276+0000 7fb8e99a1640 1 --2- 172.21.15.78:0/350373230 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fb8e4129210 0x7fb8e4132f40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:38.280 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.276+0000 7fb8e99a1640 1 -- 172.21.15.78:0/350373230 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fb8e4116680 con 0x7fb8e412ca60 2024-09-21T09:22:38.281 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.276+0000 7fb8e91a0640 1 --2- 172.21.15.78:0/350373230 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fb8e4129210 0x7fb8e4132f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:22:38.281 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.276+0000 7fb8e99a1640 1 --2- 172.21.15.78:0/350373230 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fb8e412ca60 0x7fb8e41339c0 secure :-1 s=READY pgs=53 cs=0 l=1 rev1=1 crypto rx=0x7fb8dc00c340 tx=0x7fb8dc00c800 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:38.281 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.276+0000 7fb8e1ffb640 1 -- 172.21.15.78:0/350373230 <== mon.2 v2:172.21.15.78:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb8dc011790 con 0x7fb8e412ca60 2024-09-21T09:22:38.281 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.276+0000 7fb8e1ffb640 1 -- 172.21.15.78:0/350373230 <== mon.2 v2:172.21.15.78:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fb8dc017070 con 0x7fb8e412ca60 2024-09-21T09:22:38.281 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.276+0000 7fb8ea1a2640 1 -- 172.21.15.78:0/350373230 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fb8e41162c0 con 0x7fb8e412ca60 2024-09-21T09:22:38.281 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.276+0000 7fb8e1ffb640 1 -- 172.21.15.78:0/350373230 <== mon.2 v2:172.21.15.78:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb8dc011d20 con 0x7fb8e412ca60 2024-09-21T09:22:38.281 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.276+0000 7fb8ea1a2640 1 -- 172.21.15.78:0/350373230 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7fb8e4126570 con 0x7fb8e412ca60 2024-09-21T09:22:38.281 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.277+0000 7fb8e1ffb640 1 -- 172.21.15.78:0/350373230 <== mon.2 v2:172.21.15.78:3302/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7fb8dc01c460 con 0x7fb8e412ca60 2024-09-21T09:22:38.282 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.277+0000 7fb8ea1a2640 1 -- 172.21.15.78:0/350373230 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fb8e4126800 con 0x7fb8e412ca60 2024-09-21T09:22:38.282 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.278+0000 7fb8e1ffb640 1 -- 172.21.15.78:0/350373230 <== mon.2 v2:172.21.15.78:3302/0 5 ==== osd_map(32..32 src has 1..32) ==== 3027+0+0 (secure 0 0 0) 0x7fb8dc068cf0 con 0x7fb8e412ca60 2024-09-21T09:22:38.283 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.281+0000 7fb8e1ffb640 1 -- 172.21.15.78:0/350373230 <== mon.2 v2:172.21.15.78:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7fb8e4126800 con 0x7fb8e412ca60 2024-09-21T09:22:38.429 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.427+0000 7fb8ea1a2640 1 -- 172.21.15.78:0/350373230 --> v1:172.21.15.78:6816/2186157269 -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) -- 0x7fb8e41295f0 con 0x7fb8d00514e0 2024-09-21T09:22:38.430 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.427+0000 7fb8e1ffb640 1 -- 172.21.15.78:0/350373230 <== mgr.14103 v1:172.21.15.78:6816/2186157269 1 ==== mgr_command_reply(tid 0: 0 dumped all) ==== 18+0+32014 (unknown 2965378022 0 2855338572) 0x7fb8e41295f0 con 0x7fb8d00514e0 2024-09-21T09:22:38.430 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:22:38.430 INFO:teuthology.orchestra.run.smithi078.stderr:dumped all 2024-09-21T09:22:38.432 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.429+0000 7fb8ea1a2640 1 -- 172.21.15.78:0/350373230 >> v1:172.21.15.78:6816/2186157269 conn(0x7fb8d00514e0 legacy=0x7fb8d00539a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:38.432 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.429+0000 7fb8ea1a2640 1 -- 172.21.15.78:0/350373230 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fb8e412ca60 msgr2=0x7fb8e41339c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:38.432 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.429+0000 7fb8ea1a2640 1 --2- 172.21.15.78:0/350373230 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fb8e412ca60 0x7fb8e41339c0 secure :-1 s=READY pgs=53 cs=0 l=1 rev1=1 crypto rx=0x7fb8dc00c340 tx=0x7fb8dc00c800 comp rx=0 tx=0).stop 2024-09-21T09:22:38.432 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.429+0000 7fb8ea1a2640 1 -- 172.21.15.78:0/350373230 shutdown_connections 2024-09-21T09:22:38.432 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.429+0000 7fb8ea1a2640 1 --2- 172.21.15.78:0/350373230 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fb8e412ca60 0x7fb8e41339c0 unknown :-1 s=CLOSED pgs=53 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:38.432 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.429+0000 7fb8ea1a2640 1 --2- 172.21.15.78:0/350373230 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fb8e4129b30 0x7fb8e4133480 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:38.432 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.429+0000 7fb8ea1a2640 1 --2- 172.21.15.78:0/350373230 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fb8e4129210 0x7fb8e4132f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:38.432 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.429+0000 7fb8ea1a2640 1 -- 172.21.15.78:0/350373230 >> 172.21.15.78:0/350373230 conn(0x7fb8e409d370 msgr2=0x7fb8e41348b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:38.433 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.429+0000 7fb8ea1a2640 1 -- 172.21.15.78:0/350373230 shutdown_connections 2024-09-21T09:22:38.433 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.429+0000 7fb8ea1a2640 1 -- 172.21.15.78:0/350373230 wait complete. 2024-09-21T09:22:38.442 INFO:teuthology.orchestra.run.smithi078.stdout:{"pg_ready":true,"pg_map":{"version":26,"stamp":"2024-09-21T09:22:38.402412+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":151,"num_write_kb":1896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":107,"ondisk_log_size":107,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":377487360,"kb_used":112236,"kb_used_data":2224,"kb_used_omap":17,"kb_used_meta":109934,"kb_avail":377375124,"statfs":{"total":386547056640,"available":386432126976,"internally_reserved":0,"allocated":2277376,"data_stored":1556658,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":17745,"internal_metadata":112573103},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001088"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":49,"reported_epoch":32,"state":"active+clean","last_fresh":"2024-09-21T09:22:34.490846+0000","last_change":"2024-09-21T09:22:31.543815+0000","last_active":"2024-09-21T09:22:34.490846+0000","last_peered":"2024-09-21T09:22:34.490846+0000","last_clean":"2024-09-21T09:22:34.490846+0000","last_became_active":"2024-09-21T09:22:31.543451+0000","last_became_peered":"2024-09-21T09:22:31.543451+0000","last_unstale":"2024-09-21T09:22:34.490846+0000","last_undegraded":"2024-09-21T09:22:34.490846+0000","last_fullsized":"2024-09-21T09:22:34.490846+0000","mapping_epoch":28,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":29,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:25:48.514441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039605,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":40,"reported_epoch":32,"state":"active+clean","last_fresh":"2024-09-21T09:22:34.490419+0000","last_change":"2024-09-21T09:22:29.466574+0000","last_active":"2024-09-21T09:22:34.490419+0000","last_peered":"2024-09-21T09:22:34.490419+0000","last_clean":"2024-09-21T09:22:34.490419+0000","last_became_active":"2024-09-21T09:22:29.466289+0000","last_became_peered":"2024-09-21T09:22:29.466289+0000","last_unstale":"2024-09-21T09:22:34.490419+0000","last_undegraded":"2024-09-21T09:22:34.490419+0000","last_fullsized":"2024-09-21T09:22:34.490419+0000","mapping_epoch":26,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":27,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:11:52.267440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040851400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":49,"reported_epoch":32,"state":"active+clean","last_fresh":"2024-09-21T09:22:34.490724+0000","last_change":"2024-09-21T09:22:31.543852+0000","last_active":"2024-09-21T09:22:34.490724+0000","last_peered":"2024-09-21T09:22:34.490724+0000","last_clean":"2024-09-21T09:22:34.490724+0000","last_became_active":"2024-09-21T09:22:31.543538+0000","last_became_peered":"2024-09-21T09:22:31.543538+0000","last_unstale":"2024-09-21T09:22:34.490724+0000","last_undegraded":"2024-09-21T09:22:34.490724+0000","last_fullsized":"2024-09-21T09:22:34.490724+0000","mapping_epoch":28,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":29,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:04:07.910766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001005517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":40,"reported_epoch":32,"state":"active+clean","last_fresh":"2024-09-21T09:22:34.490274+0000","last_change":"2024-09-21T09:22:29.467296+0000","last_active":"2024-09-21T09:22:34.490274+0000","last_peered":"2024-09-21T09:22:34.490274+0000","last_clean":"2024-09-21T09:22:34.490274+0000","last_became_active":"2024-09-21T09:22:29.466923+0000","last_became_peered":"2024-09-21T09:22:29.466923+0000","last_unstale":"2024-09-21T09:22:34.490274+0000","last_undegraded":"2024-09-21T09:22:34.490274+0000","last_fullsized":"2024-09-21T09:22:34.490274+0000","mapping_epoch":26,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":27,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:59:07.385986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054560899999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"17'2","reported_seq":46,"reported_epoch":32,"state":"active+clean","last_fresh":"2024-09-21T09:22:34.490675+0000","last_change":"2024-09-21T09:22:31.472650+0000","last_active":"2024-09-21T09:22:34.490675+0000","last_peered":"2024-09-21T09:22:34.490675+0000","last_clean":"2024-09-21T09:22:34.490675+0000","last_became_active":"2024-09-21T09:22:31.472357+0000","last_became_peered":"2024-09-21T09:22:31.472357+0000","last_unstale":"2024-09-21T09:22:34.490675+0000","last_undegraded":"2024-09-21T09:22:34.490675+0000","last_fullsized":"2024-09-21T09:22:34.490675+0000","mapping_epoch":28,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":29,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:07:36.189205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011881540000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":37,"reported_epoch":32,"state":"active+clean","last_fresh":"2024-09-21T09:22:34.489357+0000","last_change":"2024-09-21T09:22:31.472256+0000","last_active":"2024-09-21T09:22:34.489357+0000","last_peered":"2024-09-21T09:22:34.489357+0000","last_clean":"2024-09-21T09:22:34.489357+0000","last_became_active":"2024-09-21T09:22:31.471983+0000","last_became_peered":"2024-09-21T09:22:31.471983+0000","last_unstale":"2024-09-21T09:22:34.489357+0000","last_undegraded":"2024-09-21T09:22:34.489357+0000","last_fullsized":"2024-09-21T09:22:34.489357+0000","mapping_epoch":28,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":29,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:01:34.840425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023051899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":44,"reported_epoch":32,"state":"active+clean","last_fresh":"2024-09-21T09:22:34.490328+0000","last_change":"2024-09-21T09:22:31.473209+0000","last_active":"2024-09-21T09:22:34.490328+0000","last_peered":"2024-09-21T09:22:34.490328+0000","last_clean":"2024-09-21T09:22:34.490328+0000","last_became_active":"2024-09-21T09:22:31.472948+0000","last_became_peered":"2024-09-21T09:22:31.472948+0000","last_unstale":"2024-09-21T09:22:34.490328+0000","last_undegraded":"2024-09-21T09:22:34.490328+0000","last_fullsized":"2024-09-21T09:22:34.490328+0000","mapping_epoch":28,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":29,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:44:51.405569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039429200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"15'1","reported_seq":42,"reported_epoch":32,"state":"active+clean","last_fresh":"2024-09-21T09:22:34.490515+0000","last_change":"2024-09-21T09:22:31.472253+0000","last_active":"2024-09-21T09:22:34.490515+0000","last_peered":"2024-09-21T09:22:34.490515+0000","last_clean":"2024-09-21T09:22:34.490515+0000","last_became_active":"2024-09-21T09:22:31.471933+0000","last_became_peered":"2024-09-21T09:22:31.471933+0000","last_unstale":"2024-09-21T09:22:34.490515+0000","last_undegraded":"2024-09-21T09:22:34.490515+0000","last_fullsized":"2024-09-21T09:22:34.490515+0000","mapping_epoch":28,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":29,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:57:08.422651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091827900000000004,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"20'104","reported_seq":190,"reported_epoch":32,"state":"active+clean","last_fresh":"2024-09-21T09:22:34.490159+0000","last_change":"2024-09-21T09:22:29.466956+0000","last_active":"2024-09-21T09:22:34.490159+0000","last_peered":"2024-09-21T09:22:34.490159+0000","last_clean":"2024-09-21T09:22:34.490159+0000","last_became_active":"2024-09-21T09:22:29.466649+0000","last_became_peered":"2024-09-21T09:22:29.466649+0000","last_unstale":"2024-09-21T09:22:34.490159+0000","last_undegraded":"2024-09-21T09:22:34.490159+0000","last_fullsized":"2024-09-21T09:22:34.490159+0000","mapping_epoch":26,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":27,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:51.278939+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:51.278939+0000","last_clean_scrub_stamp":"2024-09-21T09:20:51.278939+0000","objects_scrubbed":0,"log_size":104,"log_dups_size":0,"ondisk_log_size":104,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:02:40.936727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":149,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":4},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":149,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104,"ondisk_log_size":104,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":3,"up_from":28,"seq":120259084291,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27808,"kb_used_data":268,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":94344032,"statfs":{"total":96636764160,"available":96608288768,"internally_reserved":0,"allocated":274432,"data_stored":93990,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4435,"internal_metadata":28176045},"hb_peers":[0,1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":27,"seq":115964116995,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27796,"kb_used_data":264,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":94344044,"statfs":{"total":96636764160,"available":96608301056,"internally_reserved":0,"allocated":270336,"data_stored":93971,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4437,"internal_metadata":28176043},"hb_peers":[0,1,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":1,"up_from":26,"seq":111669149700,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":28320,"kb_used_data":848,"kb_used_omap":4,"kb_used_meta":27451,"kb_avail":94343520,"statfs":{"total":96636764160,"available":96607764480,"internally_reserved":0,"allocated":868352,"data_stored":684358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4436,"internal_metadata":28110508},"hb_peers":[0,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":26,"seq":111669149700,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":28312,"kb_used_data":844,"kb_used_omap":4,"kb_used_meta":27451,"kb_avail":94343528,"statfs":{"total":96636764160,"available":96607772672,"internally_reserved":0,"allocated":864256,"data_stored":684339,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4437,"internal_metadata":28110507},"hb_peers":[1,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T09:22:38.443 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-09-21T09:22:38.545 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.542+0000 7f8dfe1f9640 1 -- 172.21.15.78:0/3549013134 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f8df812c330 msgr2=0x7f8df8130af0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:38.545 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.542+0000 7f8dfe1f9640 1 --2- 172.21.15.78:0/3549013134 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f8df812c330 0x7f8df8130af0 secure :-1 s=READY pgs=130 cs=0 l=1 rev1=1 crypto rx=0x7f8de0009830 tx=0x7f8de002a3f0 comp rx=0 tx=0).stop 2024-09-21T09:22:38.545 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.542+0000 7f8dfe1f9640 1 -- 172.21.15.78:0/3549013134 shutdown_connections 2024-09-21T09:22:38.545 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.542+0000 7f8dfe1f9640 1 --2- 172.21.15.78:0/3549013134 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f8df812c330 0x7f8df8130af0 unknown :-1 s=CLOSED pgs=130 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:38.545 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.542+0000 7f8dfe1f9640 1 --2- 172.21.15.78:0/3549013134 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f8df8131030 0x7f8df8133420 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:38.545 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.542+0000 7f8dfe1f9640 1 -- 172.21.15.78:0/3549013134 >> 172.21.15.78:0/3549013134 conn(0x7f8df809a2e0 msgr2=0x7f8df8097ad0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:38.545 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.542+0000 7f8dfe1f9640 1 -- 172.21.15.78:0/3549013134 shutdown_connections 2024-09-21T09:22:38.545 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.542+0000 7f8dfe1f9640 1 -- 172.21.15.78:0/3549013134 wait complete. 2024-09-21T09:22:38.545 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.543+0000 7f8dfe1f9640 1 Processor -- start 2024-09-21T09:22:38.548 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.543+0000 7f8dfe1f9640 1 -- start start 2024-09-21T09:22:38.549 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.543+0000 7f8dfe1f9640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f8df80991f0 0x7f8df81c4b30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:38.549 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.543+0000 7f8dfe1f9640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f8df812c330 0x7f8df81c5070 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:38.549 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.543+0000 7f8dfe1f9640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f8df8131030 0x7f8df81c55b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:38.549 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.543+0000 7f8dfe1f9640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7f8df81346b0 con 0x7f8df80991f0 2024-09-21T09:22:38.549 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.543+0000 7f8dfe1f9640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7f8df8134530 con 0x7f8df8131030 2024-09-21T09:22:38.549 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.543+0000 7f8dfe1f9640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7f8df8134830 con 0x7f8df812c330 2024-09-21T09:22:38.549 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.543+0000 7f8df77fe640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f8df80991f0 0x7f8df81c4b30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:38.549 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.543+0000 7f8df6ffd640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f8df812c330 0x7f8df81c5070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:38.549 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.543+0000 7f8df6ffd640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f8df812c330 0x7f8df81c5070 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3302/0 says I am v2:172.21.15.78:35840/0 (socket says 172.21.15.78:35840) 2024-09-21T09:22:38.549 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.543+0000 7f8df7fff640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f8df8131030 0x7f8df81c55b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:38.550 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.543+0000 7f8df77fe640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f8df80991f0 0x7f8df81c4b30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3300/0 says I am v2:172.21.15.78:44660/0 (socket says 172.21.15.78:44660) 2024-09-21T09:22:38.550 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.543+0000 7f8df6ffd640 1 -- 172.21.15.78:0/3319219567 learned_addr learned my addr 172.21.15.78:0/3319219567 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:38.550 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.543+0000 7f8df6ffd640 1 -- 172.21.15.78:0/3319219567 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f8df8131030 msgr2=0x7f8df81c55b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:38.550 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.544+0000 7f8df6ffd640 1 --2- 172.21.15.78:0/3319219567 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f8df8131030 0x7f8df81c55b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:38.550 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.544+0000 7f8df6ffd640 1 -- 172.21.15.78:0/3319219567 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f8df80991f0 msgr2=0x7f8df81c4b30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:38.550 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.544+0000 7f8df6ffd640 1 --2- 172.21.15.78:0/3319219567 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f8df80991f0 0x7f8df81c4b30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:38.550 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.544+0000 7f8df6ffd640 1 -- 172.21.15.78:0/3319219567 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f8df8094030 con 0x7f8df812c330 2024-09-21T09:22:38.550 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.544+0000 7f8df77fe640 1 --2- 172.21.15.78:0/3319219567 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f8df80991f0 0x7f8df81c4b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:22:38.550 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.544+0000 7f8df6ffd640 1 --2- 172.21.15.78:0/3319219567 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f8df812c330 0x7f8df81c5070 secure :-1 s=READY pgs=54 cs=0 l=1 rev1=1 crypto rx=0x7f8de0000c00 tx=0x7f8de0032c10 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:38.550 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.544+0000 7f8deffff640 1 -- 172.21.15.78:0/3319219567 <== mon.2 v2:172.21.15.78:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8de0040070 con 0x7f8df812c330 2024-09-21T09:22:38.550 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.544+0000 7f8deffff640 1 -- 172.21.15.78:0/3319219567 <== mon.2 v2:172.21.15.78:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f8de00038e0 con 0x7f8df812c330 2024-09-21T09:22:38.551 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.544+0000 7f8dfe1f9640 1 -- 172.21.15.78:0/3319219567 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f8df8132250 con 0x7f8df812c330 2024-09-21T09:22:38.551 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.544+0000 7f8deffff640 1 -- 172.21.15.78:0/3319219567 <== mon.2 v2:172.21.15.78:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8de0003d30 con 0x7f8df812c330 2024-09-21T09:22:38.551 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.544+0000 7f8dfe1f9640 1 -- 172.21.15.78:0/3319219567 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f8df81330c0 con 0x7f8df812c330 2024-09-21T09:22:38.551 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.545+0000 7f8deffff640 1 -- 172.21.15.78:0/3319219567 <== mon.2 v2:172.21.15.78:3302/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f8de0039040 con 0x7f8df812c330 2024-09-21T09:22:38.551 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.545+0000 7f8dfe1f9640 1 -- 172.21.15.78:0/3319219567 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f8df812bc20 con 0x7f8df812c330 2024-09-21T09:22:38.551 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.545+0000 7f8deffff640 1 -- 172.21.15.78:0/3319219567 <== mon.2 v2:172.21.15.78:3302/0 5 ==== osd_map(32..32 src has 1..32) ==== 3027+0+0 (secure 0 0 0) 0x7f8de008a3c0 con 0x7f8df812c330 2024-09-21T09:22:38.551 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.549+0000 7f8deffff640 1 -- 172.21.15.78:0/3319219567 <== mon.2 v2:172.21.15.78:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f8de0089470 con 0x7f8df812c330 2024-09-21T09:22:38.695 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.693+0000 7f8dfe1f9640 1 -- 172.21.15.78:0/3319219567 --> v1:172.21.15.78:6816/2186157269 -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) -- 0x7f8df8133380 con 0x7f8dd80514e0 2024-09-21T09:22:38.696 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.693+0000 7f8deffff640 1 -- 172.21.15.78:0/3319219567 <== mgr.14103 v1:172.21.15.78:6816/2186157269 1 ==== mgr_command_reply(tid 0: 0 dumped all) ==== 18+0+32014 (unknown 2965378022 0 2855338572) 0x7f8df8133380 con 0x7f8dd80514e0 2024-09-21T09:22:38.696 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:22:38.697 INFO:teuthology.orchestra.run.smithi078.stderr:dumped all 2024-09-21T09:22:38.698 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.695+0000 7f8dfe1f9640 1 -- 172.21.15.78:0/3319219567 >> v1:172.21.15.78:6816/2186157269 conn(0x7f8dd80514e0 legacy=0x7f8dd80539a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:38.698 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.695+0000 7f8dfe1f9640 1 -- 172.21.15.78:0/3319219567 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f8df812c330 msgr2=0x7f8df81c5070 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:38.698 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.695+0000 7f8dfe1f9640 1 --2- 172.21.15.78:0/3319219567 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f8df812c330 0x7f8df81c5070 secure :-1 s=READY pgs=54 cs=0 l=1 rev1=1 crypto rx=0x7f8de0000c00 tx=0x7f8de0032c10 comp rx=0 tx=0).stop 2024-09-21T09:22:38.698 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.695+0000 7f8dfe1f9640 1 -- 172.21.15.78:0/3319219567 shutdown_connections 2024-09-21T09:22:38.698 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.695+0000 7f8dfe1f9640 1 --2- 172.21.15.78:0/3319219567 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f8df8131030 0x7f8df81c55b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:38.698 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.695+0000 7f8dfe1f9640 1 --2- 172.21.15.78:0/3319219567 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f8df812c330 0x7f8df81c5070 unknown :-1 s=CLOSED pgs=54 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:38.698 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.695+0000 7f8dfe1f9640 1 --2- 172.21.15.78:0/3319219567 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f8df80991f0 0x7f8df81c4b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:38.699 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.695+0000 7f8dfe1f9640 1 -- 172.21.15.78:0/3319219567 >> 172.21.15.78:0/3319219567 conn(0x7f8df809a2e0 msgr2=0x7f8df81295d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:38.699 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.695+0000 7f8dfe1f9640 1 -- 172.21.15.78:0/3319219567 shutdown_connections 2024-09-21T09:22:38.699 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.696+0000 7f8dfe1f9640 1 -- 172.21.15.78:0/3319219567 wait complete. 2024-09-21T09:22:38.708 INFO:teuthology.orchestra.run.smithi078.stdout:{"pg_ready":true,"pg_map":{"version":26,"stamp":"2024-09-21T09:22:38.402412+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":151,"num_write_kb":1896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":107,"ondisk_log_size":107,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":377487360,"kb_used":112236,"kb_used_data":2224,"kb_used_omap":17,"kb_used_meta":109934,"kb_avail":377375124,"statfs":{"total":386547056640,"available":386432126976,"internally_reserved":0,"allocated":2277376,"data_stored":1556658,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":17745,"internal_metadata":112573103},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001088"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":49,"reported_epoch":32,"state":"active+clean","last_fresh":"2024-09-21T09:22:34.490846+0000","last_change":"2024-09-21T09:22:31.543815+0000","last_active":"2024-09-21T09:22:34.490846+0000","last_peered":"2024-09-21T09:22:34.490846+0000","last_clean":"2024-09-21T09:22:34.490846+0000","last_became_active":"2024-09-21T09:22:31.543451+0000","last_became_peered":"2024-09-21T09:22:31.543451+0000","last_unstale":"2024-09-21T09:22:34.490846+0000","last_undegraded":"2024-09-21T09:22:34.490846+0000","last_fullsized":"2024-09-21T09:22:34.490846+0000","mapping_epoch":28,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":29,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:25:48.514441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039605,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":40,"reported_epoch":32,"state":"active+clean","last_fresh":"2024-09-21T09:22:34.490419+0000","last_change":"2024-09-21T09:22:29.466574+0000","last_active":"2024-09-21T09:22:34.490419+0000","last_peered":"2024-09-21T09:22:34.490419+0000","last_clean":"2024-09-21T09:22:34.490419+0000","last_became_active":"2024-09-21T09:22:29.466289+0000","last_became_peered":"2024-09-21T09:22:29.466289+0000","last_unstale":"2024-09-21T09:22:34.490419+0000","last_undegraded":"2024-09-21T09:22:34.490419+0000","last_fullsized":"2024-09-21T09:22:34.490419+0000","mapping_epoch":26,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":27,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:11:52.267440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040851400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":49,"reported_epoch":32,"state":"active+clean","last_fresh":"2024-09-21T09:22:34.490724+0000","last_change":"2024-09-21T09:22:31.543852+0000","last_active":"2024-09-21T09:22:34.490724+0000","last_peered":"2024-09-21T09:22:34.490724+0000","last_clean":"2024-09-21T09:22:34.490724+0000","last_became_active":"2024-09-21T09:22:31.543538+0000","last_became_peered":"2024-09-21T09:22:31.543538+0000","last_unstale":"2024-09-21T09:22:34.490724+0000","last_undegraded":"2024-09-21T09:22:34.490724+0000","last_fullsized":"2024-09-21T09:22:34.490724+0000","mapping_epoch":28,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":29,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:04:07.910766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001005517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":40,"reported_epoch":32,"state":"active+clean","last_fresh":"2024-09-21T09:22:34.490274+0000","last_change":"2024-09-21T09:22:29.467296+0000","last_active":"2024-09-21T09:22:34.490274+0000","last_peered":"2024-09-21T09:22:34.490274+0000","last_clean":"2024-09-21T09:22:34.490274+0000","last_became_active":"2024-09-21T09:22:29.466923+0000","last_became_peered":"2024-09-21T09:22:29.466923+0000","last_unstale":"2024-09-21T09:22:34.490274+0000","last_undegraded":"2024-09-21T09:22:34.490274+0000","last_fullsized":"2024-09-21T09:22:34.490274+0000","mapping_epoch":26,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":27,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:59:07.385986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054560899999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"17'2","reported_seq":46,"reported_epoch":32,"state":"active+clean","last_fresh":"2024-09-21T09:22:34.490675+0000","last_change":"2024-09-21T09:22:31.472650+0000","last_active":"2024-09-21T09:22:34.490675+0000","last_peered":"2024-09-21T09:22:34.490675+0000","last_clean":"2024-09-21T09:22:34.490675+0000","last_became_active":"2024-09-21T09:22:31.472357+0000","last_became_peered":"2024-09-21T09:22:31.472357+0000","last_unstale":"2024-09-21T09:22:34.490675+0000","last_undegraded":"2024-09-21T09:22:34.490675+0000","last_fullsized":"2024-09-21T09:22:34.490675+0000","mapping_epoch":28,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":29,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:07:36.189205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011881540000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":37,"reported_epoch":32,"state":"active+clean","last_fresh":"2024-09-21T09:22:34.489357+0000","last_change":"2024-09-21T09:22:31.472256+0000","last_active":"2024-09-21T09:22:34.489357+0000","last_peered":"2024-09-21T09:22:34.489357+0000","last_clean":"2024-09-21T09:22:34.489357+0000","last_became_active":"2024-09-21T09:22:31.471983+0000","last_became_peered":"2024-09-21T09:22:31.471983+0000","last_unstale":"2024-09-21T09:22:34.489357+0000","last_undegraded":"2024-09-21T09:22:34.489357+0000","last_fullsized":"2024-09-21T09:22:34.489357+0000","mapping_epoch":28,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":29,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:01:34.840425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023051899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":44,"reported_epoch":32,"state":"active+clean","last_fresh":"2024-09-21T09:22:34.490328+0000","last_change":"2024-09-21T09:22:31.473209+0000","last_active":"2024-09-21T09:22:34.490328+0000","last_peered":"2024-09-21T09:22:34.490328+0000","last_clean":"2024-09-21T09:22:34.490328+0000","last_became_active":"2024-09-21T09:22:31.472948+0000","last_became_peered":"2024-09-21T09:22:31.472948+0000","last_unstale":"2024-09-21T09:22:34.490328+0000","last_undegraded":"2024-09-21T09:22:34.490328+0000","last_fullsized":"2024-09-21T09:22:34.490328+0000","mapping_epoch":28,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":29,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:44:51.405569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039429200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"15'1","reported_seq":42,"reported_epoch":32,"state":"active+clean","last_fresh":"2024-09-21T09:22:34.490515+0000","last_change":"2024-09-21T09:22:31.472253+0000","last_active":"2024-09-21T09:22:34.490515+0000","last_peered":"2024-09-21T09:22:34.490515+0000","last_clean":"2024-09-21T09:22:34.490515+0000","last_became_active":"2024-09-21T09:22:31.471933+0000","last_became_peered":"2024-09-21T09:22:31.471933+0000","last_unstale":"2024-09-21T09:22:34.490515+0000","last_undegraded":"2024-09-21T09:22:34.490515+0000","last_fullsized":"2024-09-21T09:22:34.490515+0000","mapping_epoch":28,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":29,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:57:08.422651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091827900000000004,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"20'104","reported_seq":190,"reported_epoch":32,"state":"active+clean","last_fresh":"2024-09-21T09:22:34.490159+0000","last_change":"2024-09-21T09:22:29.466956+0000","last_active":"2024-09-21T09:22:34.490159+0000","last_peered":"2024-09-21T09:22:34.490159+0000","last_clean":"2024-09-21T09:22:34.490159+0000","last_became_active":"2024-09-21T09:22:29.466649+0000","last_became_peered":"2024-09-21T09:22:29.466649+0000","last_unstale":"2024-09-21T09:22:34.490159+0000","last_undegraded":"2024-09-21T09:22:34.490159+0000","last_fullsized":"2024-09-21T09:22:34.490159+0000","mapping_epoch":26,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":27,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:51.278939+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:51.278939+0000","last_clean_scrub_stamp":"2024-09-21T09:20:51.278939+0000","objects_scrubbed":0,"log_size":104,"log_dups_size":0,"ondisk_log_size":104,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:02:40.936727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":149,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":4},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":149,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104,"ondisk_log_size":104,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":3,"up_from":28,"seq":120259084291,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27808,"kb_used_data":268,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":94344032,"statfs":{"total":96636764160,"available":96608288768,"internally_reserved":0,"allocated":274432,"data_stored":93990,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4435,"internal_metadata":28176045},"hb_peers":[0,1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":27,"seq":115964116995,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27796,"kb_used_data":264,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":94344044,"statfs":{"total":96636764160,"available":96608301056,"internally_reserved":0,"allocated":270336,"data_stored":93971,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4437,"internal_metadata":28176043},"hb_peers":[0,1,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":1,"up_from":26,"seq":111669149700,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":28320,"kb_used_data":848,"kb_used_omap":4,"kb_used_meta":27451,"kb_avail":94343520,"statfs":{"total":96636764160,"available":96607764480,"internally_reserved":0,"allocated":868352,"data_stored":684358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4436,"internal_metadata":28110508},"hb_peers":[0,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":26,"seq":111669149700,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":28312,"kb_used_data":844,"kb_used_omap":4,"kb_used_meta":27451,"kb_avail":94343528,"statfs":{"total":96636764160,"available":96607772672,"internally_reserved":0,"allocated":864256,"data_stored":684339,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4437,"internal_metadata":28110507},"hb_peers":[1,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T09:22:38.709 INFO:tasks.ceph.ceph_manager.ceph:clean! 2024-09-21T09:22:38.709 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-09-21T09:22:38.809 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.807+0000 7f175c0cb640 1 -- 172.21.15.78:0/585979397 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f1754131dc0 msgr2=0x7f17541341b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:38.810 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.807+0000 7f175c0cb640 1 --2- 172.21.15.78:0/585979397 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f1754131dc0 0x7f17541341b0 secure :-1 s=READY pgs=55 cs=0 l=1 rev1=1 crypto rx=0x7f1750009860 tx=0x7f175002a460 comp rx=0 tx=0).stop 2024-09-21T09:22:38.810 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.807+0000 7f175c0cb640 1 -- 172.21.15.78:0/585979397 shutdown_connections 2024-09-21T09:22:38.810 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.807+0000 7f175c0cb640 1 --2- 172.21.15.78:0/585979397 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f17541346f0 0x7f1754136ae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:38.810 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.807+0000 7f175c0cb640 1 --2- 172.21.15.78:0/585979397 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f1754131dc0 0x7f17541341b0 unknown :-1 s=CLOSED pgs=55 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:38.810 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.807+0000 7f175c0cb640 1 -- 172.21.15.78:0/585979397 >> 172.21.15.78:0/585979397 conn(0x7f17540a1800 msgr2=0x7f17540a1c10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:38.810 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.807+0000 7f175c0cb640 1 -- 172.21.15.78:0/585979397 shutdown_connections 2024-09-21T09:22:38.810 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.807+0000 7f175c0cb640 1 -- 172.21.15.78:0/585979397 wait complete. 2024-09-21T09:22:38.812 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.808+0000 7f175c0cb640 1 Processor -- start 2024-09-21T09:22:38.812 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.808+0000 7f175c0cb640 1 -- start start 2024-09-21T09:22:38.812 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.808+0000 7f175c0cb640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f1754099080 0x7f175412dd70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:38.812 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.808+0000 7f175c0cb640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f1754131dc0 0x7f175412e2b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:38.813 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.808+0000 7f175c0cb640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f17541346f0 0x7f175412e7f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:38.813 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.808+0000 7f175c0cb640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7f17541377a0 con 0x7f17541346f0 2024-09-21T09:22:38.813 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.808+0000 7f175c0cb640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7f1754137620 con 0x7f1754099080 2024-09-21T09:22:38.813 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.808+0000 7f175c0cb640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7f1754137920 con 0x7f1754131dc0 2024-09-21T09:22:38.813 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.808+0000 7f1759666640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f1754131dc0 0x7f175412e2b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:38.813 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.808+0000 7f1759e67640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f1754099080 0x7f175412dd70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:38.813 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.808+0000 7f175a668640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f17541346f0 0x7f175412e7f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:38.813 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.808+0000 7f1759666640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f1754131dc0 0x7f175412e2b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3302/0 says I am v2:172.21.15.78:35860/0 (socket says 172.21.15.78:35860) 2024-09-21T09:22:38.813 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.808+0000 7f1759666640 1 -- 172.21.15.78:0/3347123083 learned_addr learned my addr 172.21.15.78:0/3347123083 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:38.813 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.808+0000 7f1759666640 1 -- 172.21.15.78:0/3347123083 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f1754099080 msgr2=0x7f175412dd70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:38.813 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.808+0000 7f1759666640 1 --2- 172.21.15.78:0/3347123083 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f1754099080 0x7f175412dd70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:38.813 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.808+0000 7f1759666640 1 -- 172.21.15.78:0/3347123083 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f17541346f0 msgr2=0x7f175412e7f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:38.813 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.808+0000 7f1759666640 1 --2- 172.21.15.78:0/3347123083 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f17541346f0 0x7f175412e7f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:38.814 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.808+0000 7f1759666640 1 -- 172.21.15.78:0/3347123083 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f175409ec60 con 0x7f1754131dc0 2024-09-21T09:22:38.814 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.809+0000 7f175a668640 1 --2- 172.21.15.78:0/3347123083 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f17541346f0 0x7f175412e7f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:22:38.814 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.809+0000 7f1759e67640 1 --2- 172.21.15.78:0/3347123083 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f1754099080 0x7f175412dd70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:22:38.814 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.809+0000 7f1759666640 1 --2- 172.21.15.78:0/3347123083 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f1754131dc0 0x7f175412e2b0 secure :-1 s=READY pgs=56 cs=0 l=1 rev1=1 crypto rx=0x7f1750009780 tx=0x7f1750032b90 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:38.814 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.809+0000 7f17467fc640 1 -- 172.21.15.78:0/3347123083 <== mon.2 v2:172.21.15.78:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1750040070 con 0x7f1754131dc0 2024-09-21T09:22:38.814 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.809+0000 7f17467fc640 1 -- 172.21.15.78:0/3347123083 <== mon.2 v2:172.21.15.78:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f1750005330 con 0x7f1754131dc0 2024-09-21T09:22:38.814 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.809+0000 7f175c0cb640 1 -- 172.21.15.78:0/3347123083 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f175412ed30 con 0x7f1754131dc0 2024-09-21T09:22:38.814 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.809+0000 7f17467fc640 1 -- 172.21.15.78:0/3347123083 <== mon.2 v2:172.21.15.78:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f17500047a0 con 0x7f1754131dc0 2024-09-21T09:22:38.814 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.809+0000 7f175c0cb640 1 -- 172.21.15.78:0/3347123083 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f175412f490 con 0x7f1754131dc0 2024-09-21T09:22:38.814 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.810+0000 7f175c0cb640 1 -- 172.21.15.78:0/3347123083 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f1754137dd0 con 0x7f1754131dc0 2024-09-21T09:22:38.814 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.810+0000 7f17467fc640 1 -- 172.21.15.78:0/3347123083 <== mon.2 v2:172.21.15.78:3302/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f1750039040 con 0x7f1754131dc0 2024-09-21T09:22:38.814 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.810+0000 7f17467fc640 1 -- 172.21.15.78:0/3347123083 <== mon.2 v2:172.21.15.78:3302/0 5 ==== osd_map(32..32 src has 1..32) ==== 3027+0+0 (secure 0 0 0) 0x7f1750089e70 con 0x7f1754131dc0 2024-09-21T09:22:38.816 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.814+0000 7f17467fc640 1 -- 172.21.15.78:0/3347123083 <== mon.2 v2:172.21.15.78:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f1750088f20 con 0x7f1754131dc0 2024-09-21T09:22:38.964 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.961+0000 7f175c0cb640 1 -- 172.21.15.78:0/3347123083 --> v1:172.21.15.78:6816/2186157269 -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) -- 0x7f17540698c0 con 0x7f17380514b0 2024-09-21T09:22:38.964 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.962+0000 7f17467fc640 1 -- 172.21.15.78:0/3347123083 <== mgr.14103 v1:172.21.15.78:6816/2186157269 1 ==== mgr_command_reply(tid 0: 0 dumped all) ==== 18+0+32014 (unknown 2965378022 0 2855338572) 0x7f17540698c0 con 0x7f17380514b0 2024-09-21T09:22:38.965 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:22:38.965 INFO:teuthology.orchestra.run.smithi078.stderr:dumped all 2024-09-21T09:22:38.967 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.964+0000 7f175c0cb640 1 -- 172.21.15.78:0/3347123083 >> v1:172.21.15.78:6816/2186157269 conn(0x7f17380514b0 legacy=0x7f1738053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:38.967 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.964+0000 7f175c0cb640 1 -- 172.21.15.78:0/3347123083 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f1754131dc0 msgr2=0x7f175412e2b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:38.967 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.964+0000 7f175c0cb640 1 --2- 172.21.15.78:0/3347123083 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f1754131dc0 0x7f175412e2b0 secure :-1 s=READY pgs=56 cs=0 l=1 rev1=1 crypto rx=0x7f1750009780 tx=0x7f1750032b90 comp rx=0 tx=0).stop 2024-09-21T09:22:38.967 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.964+0000 7f175c0cb640 1 -- 172.21.15.78:0/3347123083 shutdown_connections 2024-09-21T09:22:38.967 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.964+0000 7f175c0cb640 1 --2- 172.21.15.78:0/3347123083 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f17541346f0 0x7f175412e7f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:38.967 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.964+0000 7f175c0cb640 1 --2- 172.21.15.78:0/3347123083 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f1754131dc0 0x7f175412e2b0 unknown :-1 s=CLOSED pgs=56 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:38.967 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.964+0000 7f175c0cb640 1 --2- 172.21.15.78:0/3347123083 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f1754099080 0x7f175412dd70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:38.967 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.964+0000 7f175c0cb640 1 -- 172.21.15.78:0/3347123083 >> 172.21.15.78:0/3347123083 conn(0x7f17540a1800 msgr2=0x7f17540978b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:38.967 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.964+0000 7f175c0cb640 1 -- 172.21.15.78:0/3347123083 shutdown_connections 2024-09-21T09:22:38.967 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:38.964+0000 7f175c0cb640 1 -- 172.21.15.78:0/3347123083 wait complete. 2024-09-21T09:22:38.977 INFO:teuthology.orchestra.run.smithi078.stdout:{"pg_ready":true,"pg_map":{"version":26,"stamp":"2024-09-21T09:22:38.402412+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":151,"num_write_kb":1896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":107,"ondisk_log_size":107,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":377487360,"kb_used":112236,"kb_used_data":2224,"kb_used_omap":17,"kb_used_meta":109934,"kb_avail":377375124,"statfs":{"total":386547056640,"available":386432126976,"internally_reserved":0,"allocated":2277376,"data_stored":1556658,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":17745,"internal_metadata":112573103},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001088"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":49,"reported_epoch":32,"state":"active+clean","last_fresh":"2024-09-21T09:22:34.490846+0000","last_change":"2024-09-21T09:22:31.543815+0000","last_active":"2024-09-21T09:22:34.490846+0000","last_peered":"2024-09-21T09:22:34.490846+0000","last_clean":"2024-09-21T09:22:34.490846+0000","last_became_active":"2024-09-21T09:22:31.543451+0000","last_became_peered":"2024-09-21T09:22:31.543451+0000","last_unstale":"2024-09-21T09:22:34.490846+0000","last_undegraded":"2024-09-21T09:22:34.490846+0000","last_fullsized":"2024-09-21T09:22:34.490846+0000","mapping_epoch":28,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":29,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:25:48.514441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039605,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":40,"reported_epoch":32,"state":"active+clean","last_fresh":"2024-09-21T09:22:34.490419+0000","last_change":"2024-09-21T09:22:29.466574+0000","last_active":"2024-09-21T09:22:34.490419+0000","last_peered":"2024-09-21T09:22:34.490419+0000","last_clean":"2024-09-21T09:22:34.490419+0000","last_became_active":"2024-09-21T09:22:29.466289+0000","last_became_peered":"2024-09-21T09:22:29.466289+0000","last_unstale":"2024-09-21T09:22:34.490419+0000","last_undegraded":"2024-09-21T09:22:34.490419+0000","last_fullsized":"2024-09-21T09:22:34.490419+0000","mapping_epoch":26,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":27,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:11:52.267440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040851400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":49,"reported_epoch":32,"state":"active+clean","last_fresh":"2024-09-21T09:22:34.490724+0000","last_change":"2024-09-21T09:22:31.543852+0000","last_active":"2024-09-21T09:22:34.490724+0000","last_peered":"2024-09-21T09:22:34.490724+0000","last_clean":"2024-09-21T09:22:34.490724+0000","last_became_active":"2024-09-21T09:22:31.543538+0000","last_became_peered":"2024-09-21T09:22:31.543538+0000","last_unstale":"2024-09-21T09:22:34.490724+0000","last_undegraded":"2024-09-21T09:22:34.490724+0000","last_fullsized":"2024-09-21T09:22:34.490724+0000","mapping_epoch":28,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":29,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:04:07.910766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001005517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":40,"reported_epoch":32,"state":"active+clean","last_fresh":"2024-09-21T09:22:34.490274+0000","last_change":"2024-09-21T09:22:29.467296+0000","last_active":"2024-09-21T09:22:34.490274+0000","last_peered":"2024-09-21T09:22:34.490274+0000","last_clean":"2024-09-21T09:22:34.490274+0000","last_became_active":"2024-09-21T09:22:29.466923+0000","last_became_peered":"2024-09-21T09:22:29.466923+0000","last_unstale":"2024-09-21T09:22:34.490274+0000","last_undegraded":"2024-09-21T09:22:34.490274+0000","last_fullsized":"2024-09-21T09:22:34.490274+0000","mapping_epoch":26,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":27,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:59:07.385986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054560899999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"17'2","reported_seq":46,"reported_epoch":32,"state":"active+clean","last_fresh":"2024-09-21T09:22:34.490675+0000","last_change":"2024-09-21T09:22:31.472650+0000","last_active":"2024-09-21T09:22:34.490675+0000","last_peered":"2024-09-21T09:22:34.490675+0000","last_clean":"2024-09-21T09:22:34.490675+0000","last_became_active":"2024-09-21T09:22:31.472357+0000","last_became_peered":"2024-09-21T09:22:31.472357+0000","last_unstale":"2024-09-21T09:22:34.490675+0000","last_undegraded":"2024-09-21T09:22:34.490675+0000","last_fullsized":"2024-09-21T09:22:34.490675+0000","mapping_epoch":28,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":29,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:07:36.189205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011881540000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":37,"reported_epoch":32,"state":"active+clean","last_fresh":"2024-09-21T09:22:34.489357+0000","last_change":"2024-09-21T09:22:31.472256+0000","last_active":"2024-09-21T09:22:34.489357+0000","last_peered":"2024-09-21T09:22:34.489357+0000","last_clean":"2024-09-21T09:22:34.489357+0000","last_became_active":"2024-09-21T09:22:31.471983+0000","last_became_peered":"2024-09-21T09:22:31.471983+0000","last_unstale":"2024-09-21T09:22:34.489357+0000","last_undegraded":"2024-09-21T09:22:34.489357+0000","last_fullsized":"2024-09-21T09:22:34.489357+0000","mapping_epoch":28,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":29,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:01:34.840425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023051899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":44,"reported_epoch":32,"state":"active+clean","last_fresh":"2024-09-21T09:22:34.490328+0000","last_change":"2024-09-21T09:22:31.473209+0000","last_active":"2024-09-21T09:22:34.490328+0000","last_peered":"2024-09-21T09:22:34.490328+0000","last_clean":"2024-09-21T09:22:34.490328+0000","last_became_active":"2024-09-21T09:22:31.472948+0000","last_became_peered":"2024-09-21T09:22:31.472948+0000","last_unstale":"2024-09-21T09:22:34.490328+0000","last_undegraded":"2024-09-21T09:22:34.490328+0000","last_fullsized":"2024-09-21T09:22:34.490328+0000","mapping_epoch":28,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":29,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:44:51.405569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039429200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"15'1","reported_seq":42,"reported_epoch":32,"state":"active+clean","last_fresh":"2024-09-21T09:22:34.490515+0000","last_change":"2024-09-21T09:22:31.472253+0000","last_active":"2024-09-21T09:22:34.490515+0000","last_peered":"2024-09-21T09:22:34.490515+0000","last_clean":"2024-09-21T09:22:34.490515+0000","last_became_active":"2024-09-21T09:22:31.471933+0000","last_became_peered":"2024-09-21T09:22:31.471933+0000","last_unstale":"2024-09-21T09:22:34.490515+0000","last_undegraded":"2024-09-21T09:22:34.490515+0000","last_fullsized":"2024-09-21T09:22:34.490515+0000","mapping_epoch":28,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":29,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:57:08.422651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091827900000000004,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"20'104","reported_seq":190,"reported_epoch":32,"state":"active+clean","last_fresh":"2024-09-21T09:22:34.490159+0000","last_change":"2024-09-21T09:22:29.466956+0000","last_active":"2024-09-21T09:22:34.490159+0000","last_peered":"2024-09-21T09:22:34.490159+0000","last_clean":"2024-09-21T09:22:34.490159+0000","last_became_active":"2024-09-21T09:22:29.466649+0000","last_became_peered":"2024-09-21T09:22:29.466649+0000","last_unstale":"2024-09-21T09:22:34.490159+0000","last_undegraded":"2024-09-21T09:22:34.490159+0000","last_fullsized":"2024-09-21T09:22:34.490159+0000","mapping_epoch":26,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":27,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:51.278939+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:51.278939+0000","last_clean_scrub_stamp":"2024-09-21T09:20:51.278939+0000","objects_scrubbed":0,"log_size":104,"log_dups_size":0,"ondisk_log_size":104,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:02:40.936727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":149,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":4},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":149,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104,"ondisk_log_size":104,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":3,"up_from":28,"seq":120259084291,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27808,"kb_used_data":268,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":94344032,"statfs":{"total":96636764160,"available":96608288768,"internally_reserved":0,"allocated":274432,"data_stored":93990,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4435,"internal_metadata":28176045},"hb_peers":[0,1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":27,"seq":115964116995,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27796,"kb_used_data":264,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":94344044,"statfs":{"total":96636764160,"available":96608301056,"internally_reserved":0,"allocated":270336,"data_stored":93971,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4437,"internal_metadata":28176043},"hb_peers":[0,1,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":1,"up_from":26,"seq":111669149700,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":28320,"kb_used_data":848,"kb_used_omap":4,"kb_used_meta":27451,"kb_avail":94343520,"statfs":{"total":96636764160,"available":96607764480,"internally_reserved":0,"allocated":868352,"data_stored":684358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4436,"internal_metadata":28110508},"hb_peers":[0,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":26,"seq":111669149700,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":28312,"kb_used_data":844,"kb_used_omap":4,"kb_used_meta":27451,"kb_avail":94343528,"statfs":{"total":96636764160,"available":96607772672,"internally_reserved":0,"allocated":864256,"data_stored":684339,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4437,"internal_metadata":28110507},"hb_peers":[1,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T09:22:38.978 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-09-21T09:22:39.079 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:39.076+0000 7fcffa652640 1 -- 172.21.15.78:0/798935902 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fcff40991f0 msgr2=0x7fcff40910d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:39.079 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:39.076+0000 7fcffa652640 1 --2- 172.21.15.78:0/798935902 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fcff40991f0 0x7fcff40910d0 secure :-1 s=READY pgs=57 cs=0 l=1 rev1=1 crypto rx=0x7fcfe4005fd0 tx=0x7fcfe402a410 comp rx=0 tx=0).stop 2024-09-21T09:22:39.079 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:39.076+0000 7fcffa652640 1 -- 172.21.15.78:0/798935902 shutdown_connections 2024-09-21T09:22:39.079 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:39.076+0000 7fcffa652640 1 --2- 172.21.15.78:0/798935902 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fcff412f260 0x7fcff412f6f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:39.079 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:39.076+0000 7fcffa652640 1 --2- 172.21.15.78:0/798935902 >> v2:172.21.15.78:3300/0 conn(0x7fcff4091610 0x7fcff4091a30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:39.080 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:39.076+0000 7fcffa652640 1 --2- 172.21.15.78:0/798935902 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fcff40991f0 0x7fcff40910d0 unknown :-1 s=CLOSED pgs=57 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:39.080 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:39.076+0000 7fcffa652640 1 -- 172.21.15.78:0/798935902 >> 172.21.15.78:0/798935902 conn(0x7fcff409a2e0 msgr2=0x7fcff4097ad0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:39.080 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:39.076+0000 7fcffa652640 1 -- 172.21.15.78:0/798935902 shutdown_connections 2024-09-21T09:22:39.080 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:39.076+0000 7fcffa652640 1 -- 172.21.15.78:0/798935902 wait complete. 2024-09-21T09:22:39.080 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:39.077+0000 7fcffa652640 1 Processor -- start 2024-09-21T09:22:39.080 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:39.077+0000 7fcffa652640 1 -- start start 2024-09-21T09:22:39.082 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:39.077+0000 7fcffa652640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fcff4091610 0x7fcff41c5120 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:39.082 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:39.077+0000 7fcffa652640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fcff40991f0 0x7fcff41c5660 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:39.082 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:39.077+0000 7fcffa652640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fcff412f260 0x7fcff4070280 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:39.082 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:39.077+0000 7fcffa652640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7fcff4136ab0 con 0x7fcff4091610 2024-09-21T09:22:39.082 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:39.077+0000 7fcffa652640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7fcff4136930 con 0x7fcff40991f0 2024-09-21T09:22:39.082 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:39.077+0000 7fcffa652640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7fcff4136c30 con 0x7fcff412f260 2024-09-21T09:22:39.082 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:39.077+0000 7fcff3fff640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fcff4091610 0x7fcff41c5120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:39.082 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:39.077+0000 7fcff37fe640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fcff40991f0 0x7fcff41c5660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:39.082 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:39.077+0000 7fcff8bef640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fcff412f260 0x7fcff4070280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:39.082 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:39.077+0000 7fcff3fff640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fcff4091610 0x7fcff41c5120 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3300/0 says I am v2:172.21.15.78:44690/0 (socket says 172.21.15.78:44690) 2024-09-21T09:22:39.083 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:39.077+0000 7fcff3fff640 1 -- 172.21.15.78:0/140477615 learned_addr learned my addr 172.21.15.78:0/140477615 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:39.083 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:39.078+0000 7fcff3fff640 1 -- 172.21.15.78:0/140477615 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fcff412f260 msgr2=0x7fcff4070280 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:39.083 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:39.078+0000 7fcff3fff640 1 --2- 172.21.15.78:0/140477615 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fcff412f260 0x7fcff4070280 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:39.083 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:39.078+0000 7fcff3fff640 1 -- 172.21.15.78:0/140477615 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fcff40991f0 msgr2=0x7fcff41c5660 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:39.083 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:39.078+0000 7fcff3fff640 1 --2- 172.21.15.78:0/140477615 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fcff40991f0 0x7fcff41c5660 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:39.083 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:39.078+0000 7fcff3fff640 1 -- 172.21.15.78:0/140477615 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fcff4093380 con 0x7fcff4091610 2024-09-21T09:22:39.083 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:39.078+0000 7fcff8bef640 1 --2- 172.21.15.78:0/140477615 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fcff412f260 0x7fcff4070280 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:22:39.083 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:39.078+0000 7fcff37fe640 1 --2- 172.21.15.78:0/140477615 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fcff40991f0 0x7fcff41c5660 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:22:39.083 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:39.078+0000 7fcff3fff640 1 --2- 172.21.15.78:0/140477615 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fcff4091610 0x7fcff41c5120 secure :-1 s=READY pgs=132 cs=0 l=1 rev1=1 crypto rx=0x7fcfe4006010 tx=0x7fcfe40038a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:39.083 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:39.078+0000 7fcff0ff9640 1 -- 172.21.15.78:0/140477615 <== mon.0 v2:172.21.15.78:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fcfe403f070 con 0x7fcff4091610 2024-09-21T09:22:39.083 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:39.078+0000 7fcff0ff9640 1 -- 172.21.15.78:0/140477615 <== mon.0 v2:172.21.15.78:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fcfe40047a0 con 0x7fcff4091610 2024-09-21T09:22:39.083 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:39.078+0000 7fcffa652640 1 -- 172.21.15.78:0/140477615 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fcff4072880 con 0x7fcff4091610 2024-09-21T09:22:39.083 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:39.078+0000 7fcff0ff9640 1 -- 172.21.15.78:0/140477615 <== mon.0 v2:172.21.15.78:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fcfe4035030 con 0x7fcff4091610 2024-09-21T09:22:39.083 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:39.078+0000 7fcffa652640 1 -- 172.21.15.78:0/140477615 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fcff41c5ba0 con 0x7fcff4091610 2024-09-21T09:22:39.084 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:39.079+0000 7fcffa652640 1 -- 172.21.15.78:0/140477615 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fcff40910d0 con 0x7fcff4091610 2024-09-21T09:22:39.084 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:39.079+0000 7fcff0ff9640 1 -- 172.21.15.78:0/140477615 <== mon.0 v2:172.21.15.78:3300/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7fcfe4042070 con 0x7fcff4091610 2024-09-21T09:22:39.084 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:39.080+0000 7fcff0ff9640 1 -- 172.21.15.78:0/140477615 <== mon.0 v2:172.21.15.78:3300/0 5 ==== osd_map(32..32 src has 1..32) ==== 3027+0+0 (secure 0 0 0) 0x7fcfe4089f90 con 0x7fcff4091610 2024-09-21T09:22:39.085 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:39.083+0000 7fcff0ff9640 1 -- 172.21.15.78:0/140477615 <== mon.0 v2:172.21.15.78:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7fcff4093380 con 0x7fcff4091610 2024-09-21T09:22:39.235 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:39.233+0000 7fcffa652640 1 -- 172.21.15.78:0/140477615 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7fcff4074690 con 0x7fcff4091610 2024-09-21T09:22:39.235 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:39.233+0000 7fcff0ff9640 1 -- 172.21.15.78:0/140477615 <== mon.0 v2:172.21.15.78:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v32) ==== 74+0+9175 (secure 0 0 0) 0x7fcfe4089040 con 0x7fcff4091610 2024-09-21T09:22:39.236 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:22:39.236 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":32,"fsid":"3633ad52-1063-4406-97d9-d53c339f43ca","created":"2024-09-21T09:20:38.248415+0000","modified":"2024-09-21T09:22:34.481410+0000","last_up_change":"2024-09-21T09:22:30.459558+0000","last_in_change":"2024-09-21T09:20:40.330925+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":7,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":4,"require_min_compat_client":"reef","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T09:20:51.169034+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"31","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-21T09:20:54.918007+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"32","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":17,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"6b18d222-0705-4495-a01c-3e2ba2ab4cff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":9,"last_clean_end":20,"up_from":26,"up_thru":26,"down_at":21,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6800","nonce":2538472627}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6801","nonce":2538472627}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6805","nonce":2538472627}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6803","nonce":2538472627}]},"public_addr":"172.21.15.78:6800/2538472627","cluster_addr":"172.21.15.78:6801/2538472627","heartbeat_back_addr":"172.21.15.78:6805/2538472627","heartbeat_front_addr":"172.21.15.78:6803/2538472627","state":["exists","up"]},{"osd":1,"uuid":"2d15b0fa-66c2-4bda-b64b-d72cbbb16fb1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":9,"last_clean_end":20,"up_from":26,"up_thru":26,"down_at":22,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6802","nonce":3598780309}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6804","nonce":3598780309}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6807","nonce":3598780309}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6806","nonce":3598780309}]},"public_addr":"172.21.15.78:6802/3598780309","cluster_addr":"172.21.15.78:6804/3598780309","heartbeat_back_addr":"172.21.15.78:6807/3598780309","heartbeat_front_addr":"172.21.15.78:6806/3598780309","state":["exists","up"]},{"osd":2,"uuid":"0bf75d8f-017d-488d-8a2e-c3fa3b2a29c2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":9,"last_clean_end":22,"up_from":27,"up_thru":28,"down_at":23,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6808","nonce":2733362473}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6809","nonce":2733362473}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6811","nonce":2733362473}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6810","nonce":2733362473}]},"public_addr":"172.21.15.78:6808/2733362473","cluster_addr":"172.21.15.78:6809/2733362473","heartbeat_back_addr":"172.21.15.78:6811/2733362473","heartbeat_front_addr":"172.21.15.78:6810/2733362473","state":["exists","up"]},{"osd":3,"uuid":"5e3788c8-b89a-4614-b3ec-7ee1250ddbe0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":9,"last_clean_end":23,"up_from":28,"up_thru":28,"down_at":24,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6812","nonce":3499964365}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6813","nonce":3499964365}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6815","nonce":3499964365}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.78:6814","nonce":3499964365}]},"public_addr":"172.21.15.78:6812/3499964365","cluster_addr":"172.21.15.78:6813/3499964365","heartbeat_back_addr":"172.21.15.78:6815/3499964365","heartbeat_front_addr":"172.21.15.78:6814/3499964365","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:20:45.499319+0000","dead_epoch":21},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:20:45.489695+0000","dead_epoch":22},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:20:45.541959+0000","dead_epoch":23},{"osd":3,"down_stamp":"2024-09-21T09:22:23.923196+0000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:20:45.476915+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.78:6816/2602406893":"2024-09-22T09:22:16.387689+0000","172.21.15.78:0/1812672186":"2024-09-22T09:22:16.387689+0000","172.21.15.78:0/2894754949":"2024-09-22T09:22:16.387689+0000","172.21.15.78:0/2078474014":"2024-09-22T09:22:16.387689+0000","172.21.15.78:0/2453110290":"2024-09-22T09:22:16.387689+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T09:22:39.238 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:39.235+0000 7fcffa652640 1 -- 172.21.15.78:0/140477615 >> v1:172.21.15.78:6816/2186157269 conn(0x7fcfe00514e0 legacy=0x7fcfe00539a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:39.238 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:39.235+0000 7fcffa652640 1 -- 172.21.15.78:0/140477615 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fcff4091610 msgr2=0x7fcff41c5120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:39.238 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:39.235+0000 7fcffa652640 1 --2- 172.21.15.78:0/140477615 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fcff4091610 0x7fcff41c5120 secure :-1 s=READY pgs=132 cs=0 l=1 rev1=1 crypto rx=0x7fcfe4006010 tx=0x7fcfe40038a0 comp rx=0 tx=0).stop 2024-09-21T09:22:39.238 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:39.235+0000 7fcffa652640 1 -- 172.21.15.78:0/140477615 shutdown_connections 2024-09-21T09:22:39.238 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:39.235+0000 7fcffa652640 1 --2- 172.21.15.78:0/140477615 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fcff412f260 0x7fcff4070280 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:39.238 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:39.235+0000 7fcffa652640 1 --2- 172.21.15.78:0/140477615 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fcff40991f0 0x7fcff41c5660 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:39.238 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:39.235+0000 7fcffa652640 1 --2- 172.21.15.78:0/140477615 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fcff4091610 0x7fcff41c5120 unknown :-1 s=CLOSED pgs=132 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:39.238 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:39.235+0000 7fcffa652640 1 -- 172.21.15.78:0/140477615 >> 172.21.15.78:0/140477615 conn(0x7fcff409a2e0 msgr2=0x7fcff409aed0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:39.238 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:39.235+0000 7fcffa652640 1 -- 172.21.15.78:0/140477615 shutdown_connections 2024-09-21T09:22:39.238 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:39.235+0000 7fcffa652640 1 -- 172.21.15.78:0/140477615 wait complete. 2024-09-21T09:22:40.249 INFO:tasks.ceph:Scrubbing osd.0 2024-09-21T09:22:40.249 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.0 config set osd_debug_deep_scrub_sleep 0 2024-09-21T09:22:40.364 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.361+0000 7f01b207a640 1 -- 172.21.15.78:0/4050158774 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f01ac12eaa0 msgr2=0x7f01ac130e90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:40.364 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.361+0000 7f01b207a640 1 --2- 172.21.15.78:0/4050158774 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f01ac12eaa0 0x7f01ac130e90 secure :-1 s=READY pgs=58 cs=0 l=1 rev1=1 crypto rx=0x7f01a0009830 tx=0x7f01a002a3f0 comp rx=0 tx=0).stop 2024-09-21T09:22:40.364 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.362+0000 7f01b207a640 1 -- 172.21.15.78:0/4050158774 shutdown_connections 2024-09-21T09:22:40.364 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.362+0000 7f01b207a640 1 --2- 172.21.15.78:0/4050158774 >> v2:172.21.15.78:3300/0 conn(0x7f01ac1313d0 0x7f01ac1337c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:40.365 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.362+0000 7f01b207a640 1 --2- 172.21.15.78:0/4050158774 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f01ac12eaa0 0x7f01ac130e90 unknown :-1 s=CLOSED pgs=58 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:40.365 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.362+0000 7f01b207a640 1 -- 172.21.15.78:0/4050158774 >> 172.21.15.78:0/4050158774 conn(0x7f01ac09a3f0 msgr2=0x7f01ac097bd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:40.365 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.362+0000 7f01b207a640 1 -- 172.21.15.78:0/4050158774 shutdown_connections 2024-09-21T09:22:40.365 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.362+0000 7f01b207a640 1 -- 172.21.15.78:0/4050158774 wait complete. 2024-09-21T09:22:40.367 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.362+0000 7f01b207a640 1 Processor -- start 2024-09-21T09:22:40.367 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.363+0000 7f01b207a640 1 -- start start 2024-09-21T09:22:40.367 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.363+0000 7f01b207a640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f01ac12bc10 0x7f01ac08bf80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:40.367 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.363+0000 7f01b207a640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f01ac12eaa0 0x7f01ac08c4c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:40.367 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.363+0000 7f01b207a640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f01ac1313d0 0x7f01ac08ca00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:40.367 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.363+0000 7f01b207a640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7f01ac134750 con 0x7f01ac12eaa0 2024-09-21T09:22:40.367 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.363+0000 7f01b207a640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7f01ac1345d0 con 0x7f01ac12bc10 2024-09-21T09:22:40.367 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.363+0000 7f01b207a640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7f01ac1348d0 con 0x7f01ac1313d0 2024-09-21T09:22:40.367 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.363+0000 7f01ab7fe640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f01ac12bc10 0x7f01ac08bf80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:40.367 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.363+0000 7f01aaffd640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f01ac12eaa0 0x7f01ac08c4c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:40.367 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.363+0000 7f01aaffd640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f01ac12eaa0 0x7f01ac08c4c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3300/0 says I am v2:172.21.15.78:44700/0 (socket says 172.21.15.78:44700) 2024-09-21T09:22:40.368 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.363+0000 7f01aaffd640 1 -- 172.21.15.78:0/242944100 learned_addr learned my addr 172.21.15.78:0/242944100 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:40.368 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.363+0000 7f01abfff640 1 --2- 172.21.15.78:0/242944100 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f01ac1313d0 0x7f01ac08ca00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:40.368 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.363+0000 7f01aaffd640 1 -- 172.21.15.78:0/242944100 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f01ac1313d0 msgr2=0x7f01ac08ca00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:40.368 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.363+0000 7f01aaffd640 1 --2- 172.21.15.78:0/242944100 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f01ac1313d0 0x7f01ac08ca00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:40.368 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.363+0000 7f01aaffd640 1 -- 172.21.15.78:0/242944100 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f01ac12bc10 msgr2=0x7f01ac08bf80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:40.368 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.363+0000 7f01aaffd640 1 --2- 172.21.15.78:0/242944100 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f01ac12bc10 0x7f01ac08bf80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:40.368 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.363+0000 7f01aaffd640 1 -- 172.21.15.78:0/242944100 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f01ac08d6a0 con 0x7f01ac12eaa0 2024-09-21T09:22:40.368 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.363+0000 7f01abfff640 1 --2- 172.21.15.78:0/242944100 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f01ac1313d0 0x7f01ac08ca00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:22:40.368 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.363+0000 7f01ab7fe640 1 --2- 172.21.15.78:0/242944100 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f01ac12bc10 0x7f01ac08bf80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:22:40.368 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.364+0000 7f01aaffd640 1 --2- 172.21.15.78:0/242944100 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f01ac12eaa0 0x7f01ac08c4c0 secure :-1 s=READY pgs=134 cs=0 l=1 rev1=1 crypto rx=0x7f01a0000c00 tx=0x7f01a0032c10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:40.368 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.364+0000 7f0193fff640 1 -- 172.21.15.78:0/242944100 <== mon.0 v2:172.21.15.78:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f01a0040070 con 0x7f01ac12eaa0 2024-09-21T09:22:40.368 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.364+0000 7f0193fff640 1 -- 172.21.15.78:0/242944100 <== mon.0 v2:172.21.15.78:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f01a00038e0 con 0x7f01ac12eaa0 2024-09-21T09:22:40.369 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.364+0000 7f01b207a640 1 -- 172.21.15.78:0/242944100 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f01ac08ea60 con 0x7f01ac12eaa0 2024-09-21T09:22:40.369 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.364+0000 7f0193fff640 1 -- 172.21.15.78:0/242944100 <== mon.0 v2:172.21.15.78:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f01a0003d30 con 0x7f01ac12eaa0 2024-09-21T09:22:40.369 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.364+0000 7f01b207a640 1 -- 172.21.15.78:0/242944100 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f01ac0948b0 con 0x7f01ac12eaa0 2024-09-21T09:22:40.369 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.365+0000 7f01b207a640 1 -- 172.21.15.78:0/242944100 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_get_version(what=osdmap handle=1) -- 0x7f01ac093fa0 con 0x7f01ac12eaa0 2024-09-21T09:22:40.369 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.365+0000 7f0193fff640 1 -- 172.21.15.78:0/242944100 <== mon.0 v2:172.21.15.78:3300/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f01a0039040 con 0x7f01ac12eaa0 2024-09-21T09:22:40.370 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.365+0000 7f0193fff640 1 -- 172.21.15.78:0/242944100 <== mon.0 v2:172.21.15.78:3300/0 5 ==== osd_map(32..32 src has 1..32) ==== 3027+0+0 (secure 0 0 0) 0x7f01a008a3f0 con 0x7f01ac12eaa0 2024-09-21T09:22:40.370 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.365+0000 7f0193fff640 1 -- 172.21.15.78:0/242944100 --> v1:172.21.15.78:6800/2538472627 -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7f019805b9e0 con 0x7f0198057d30 2024-09-21T09:22:40.370 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.366+0000 7f0193fff640 1 -- 172.21.15.78:0/242944100 <== mon.0 v2:172.21.15.78:3300/0 6 ==== mon_get_version_reply(handle=1 version=32) ==== 24+0+0 (secure 0 0 0) 0x7f01a0091050 con 0x7f01ac12eaa0 2024-09-21T09:22:40.370 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.366+0000 7f0193fff640 1 -- 172.21.15.78:0/242944100 <== osd.0 v1:172.21.15.78:6800/2538472627 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (unknown 0 0 1302949928) 0x7f019805b9e0 con 0x7f0198057d30 2024-09-21T09:22:40.381 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.379+0000 7f01b207a640 1 -- 172.21.15.78:0/242944100 --> v1:172.21.15.78:6800/2538472627 -- command(tid 2: {"prefix": "config set", "var": "osd_debug_deep_scrub_sleep", "val": ["0"]}) -- 0x7f01ac09ca20 con 0x7f0198057d30 2024-09-21T09:22:40.381 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.379+0000 7f0193fff640 1 -- 172.21.15.78:0/242944100 <== osd.0 v1:172.21.15.78:6800/2538472627 2 ==== command_reply(tid 2: 0 ) ==== 8+0+565 (unknown 0 0 125856537) 0x7f01ac09ca20 con 0x7f0198057d30 2024-09-21T09:22:40.382 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-21T09:22:40.382 INFO:teuthology.orchestra.run.smithi078.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' " 2024-09-21T09:22:40.382 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-21T09:22:40.383 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.379+0000 7f01b207a640 1 -- 172.21.15.78:0/242944100 >> v1:172.21.15.78:6800/2538472627 conn(0x7f0198057d30 legacy=0x7f019805a170 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:40.383 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.379+0000 7f01b207a640 1 -- 172.21.15.78:0/242944100 >> v1:172.21.15.78:6816/2186157269 conn(0x7f01980514b0 legacy=0x7f0198053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:40.383 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.379+0000 7f01b207a640 1 -- 172.21.15.78:0/242944100 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f01ac12eaa0 msgr2=0x7f01ac08c4c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:40.383 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.379+0000 7f01b207a640 1 --2- 172.21.15.78:0/242944100 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f01ac12eaa0 0x7f01ac08c4c0 secure :-1 s=READY pgs=134 cs=0 l=1 rev1=1 crypto rx=0x7f01a0000c00 tx=0x7f01a0032c10 comp rx=0 tx=0).stop 2024-09-21T09:22:40.383 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.379+0000 7f01abfff640 1 -- 172.21.15.78:0/242944100 reap_dead start 2024-09-21T09:22:40.383 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.379+0000 7f01b207a640 1 -- 172.21.15.78:0/242944100 shutdown_connections 2024-09-21T09:22:40.383 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.379+0000 7f01b207a640 1 -- 172.21.15.78:0/242944100 >> 172.21.15.78:0/242944100 conn(0x7f01ac09a3f0 msgr2=0x7f01ac09c090 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:40.383 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.380+0000 7f01b207a640 1 -- 172.21.15.78:0/242944100 shutdown_connections 2024-09-21T09:22:40.383 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.380+0000 7f01b207a640 1 -- 172.21.15.78:0/242944100 wait complete. 2024-09-21T09:22:40.392 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd deep-scrub 0 2024-09-21T09:22:40.493 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.490+0000 7f61d843c640 1 -- 172.21.15.78:0/1100750076 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f61d0098d40 msgr2=0x7f61d0099120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:40.493 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.490+0000 7f61d843c640 1 --2- 172.21.15.78:0/1100750076 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f61d0098d40 0x7f61d0099120 secure :-1 s=READY pgs=59 cs=0 l=1 rev1=1 crypto rx=0x7f61cc009830 tx=0x7f61cc02a360 comp rx=0 tx=0).stop 2024-09-21T09:22:40.493 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.490+0000 7f61d843c640 1 -- 172.21.15.78:0/1100750076 shutdown_connections 2024-09-21T09:22:40.493 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.490+0000 7f61d843c640 1 --2- 172.21.15.78:0/1100750076 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f61d012eff0 0x7f61d012f480 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:40.493 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.490+0000 7f61d843c640 1 --2- 172.21.15.78:0/1100750076 >> v2:172.21.15.78:3300/0 conn(0x7f61d0099660 0x7f61d012eab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:40.493 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.490+0000 7f61d843c640 1 --2- 172.21.15.78:0/1100750076 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f61d0098d40 0x7f61d0099120 unknown :-1 s=CLOSED pgs=59 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:40.493 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.490+0000 7f61d843c640 1 -- 172.21.15.78:0/1100750076 >> 172.21.15.78:0/1100750076 conn(0x7f61d00a1790 msgr2=0x7f61d00a1ba0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:40.493 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.491+0000 7f61d843c640 1 -- 172.21.15.78:0/1100750076 shutdown_connections 2024-09-21T09:22:40.494 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.491+0000 7f61d843c640 1 -- 172.21.15.78:0/1100750076 wait complete. 2024-09-21T09:22:40.495 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.491+0000 7f61d843c640 1 Processor -- start 2024-09-21T09:22:40.496 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.491+0000 7f61d843c640 1 -- start start 2024-09-21T09:22:40.496 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.492+0000 7f61d843c640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f61d0098d40 0x7f61d01c4cb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:40.496 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.492+0000 7f61d843c640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f61d0099660 0x7f61d01c51f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:40.496 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.492+0000 7f61d843c640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f61d012eff0 0x7f61d00713d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:40.496 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.492+0000 7f61d843c640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7f61d01365b0 con 0x7f61d0099660 2024-09-21T09:22:40.496 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.492+0000 7f61d843c640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7f61d0136430 con 0x7f61d012eff0 2024-09-21T09:22:40.496 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.492+0000 7f61d843c640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7f61d0136730 con 0x7f61d0098d40 2024-09-21T09:22:40.496 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.492+0000 7f61d59d7640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f61d0099660 0x7f61d01c51f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:40.496 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.492+0000 7f61d61d8640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f61d0098d40 0x7f61d01c4cb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:40.496 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.492+0000 7f61d61d8640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f61d0098d40 0x7f61d01c4cb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3302/0 says I am v2:172.21.15.78:35894/0 (socket says 172.21.15.78:35894) 2024-09-21T09:22:40.496 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.492+0000 7f61d69d9640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f61d012eff0 0x7f61d00713d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:40.496 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.492+0000 7f61d59d7640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f61d0099660 0x7f61d01c51f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3300/0 says I am v2:172.21.15.78:44716/0 (socket says 172.21.15.78:44716) 2024-09-21T09:22:40.497 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.492+0000 7f61d61d8640 1 -- 172.21.15.78:0/2167494167 learned_addr learned my addr 172.21.15.78:0/2167494167 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:40.497 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.492+0000 7f61d61d8640 1 -- 172.21.15.78:0/2167494167 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f61d012eff0 msgr2=0x7f61d00713d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:40.497 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.492+0000 7f61d61d8640 1 --2- 172.21.15.78:0/2167494167 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f61d012eff0 0x7f61d00713d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:40.497 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.492+0000 7f61d61d8640 1 -- 172.21.15.78:0/2167494167 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f61d0099660 msgr2=0x7f61d01c51f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:40.497 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.492+0000 7f61d61d8640 1 --2- 172.21.15.78:0/2167494167 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f61d0099660 0x7f61d01c51f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:40.497 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.492+0000 7f61d61d8640 1 -- 172.21.15.78:0/2167494167 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f61d0072d20 con 0x7f61d0098d40 2024-09-21T09:22:40.497 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.492+0000 7f61d59d7640 1 --2- 172.21.15.78:0/2167494167 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f61d0099660 0x7f61d01c51f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:22:40.497 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.492+0000 7f61d61d8640 1 --2- 172.21.15.78:0/2167494167 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f61d0098d40 0x7f61d01c4cb0 secure :-1 s=READY pgs=58 cs=0 l=1 rev1=1 crypto rx=0x7f61cc009780 tx=0x7f61cc033b70 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:40.497 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.492+0000 7f61c6ffd640 1 -- 172.21.15.78:0/2167494167 <== mon.2 v2:172.21.15.78:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f61cc03f020 con 0x7f61d0098d40 2024-09-21T09:22:40.497 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.493+0000 7f61c6ffd640 1 -- 172.21.15.78:0/2167494167 <== mon.2 v2:172.21.15.78:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f61cc033d60 con 0x7f61d0098d40 2024-09-21T09:22:40.497 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.493+0000 7f61d843c640 1 -- 172.21.15.78:0/2167494167 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f61d01c5730 con 0x7f61d0098d40 2024-09-21T09:22:40.497 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.493+0000 7f61c6ffd640 1 -- 172.21.15.78:0/2167494167 <== mon.2 v2:172.21.15.78:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f61cc037040 con 0x7f61d0098d40 2024-09-21T09:22:40.497 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.493+0000 7f61d843c640 1 -- 172.21.15.78:0/2167494167 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f61d0073fe0 con 0x7f61d0098d40 2024-09-21T09:22:40.497 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.494+0000 7f61d843c640 1 -- 172.21.15.78:0/2167494167 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f61d0099120 con 0x7f61d0098d40 2024-09-21T09:22:40.498 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.494+0000 7f61c6ffd640 1 -- 172.21.15.78:0/2167494167 <== mon.2 v2:172.21.15.78:3302/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f61cc042020 con 0x7f61d0098d40 2024-09-21T09:22:40.498 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.494+0000 7f61c6ffd640 1 -- 172.21.15.78:0/2167494167 <== mon.2 v2:172.21.15.78:3302/0 5 ==== osd_map(32..32 src has 1..32) ==== 3027+0+0 (secure 0 0 0) 0x7f61cc088dc0 con 0x7f61d0098d40 2024-09-21T09:22:40.500 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.498+0000 7f61c6ffd640 1 -- 172.21.15.78:0/2167494167 <== mon.2 v2:172.21.15.78:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f61cc049030 con 0x7f61d0098d40 2024-09-21T09:22:40.649 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.647+0000 7f61d843c640 1 -- 172.21.15.78:0/2167494167 --> v1:172.21.15.78:6816/2186157269 -- mgr_command(tid 0: {"prefix": "osd deep-scrub", "who": "0", "target": ["mon-mgr", ""]}) -- 0x7f61d00698c0 con 0x7f61b40514b0 2024-09-21T09:22:40.649 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.647+0000 7f61c6ffd640 1 -- 172.21.15.78:0/2167494167 <== mgr.14103 v1:172.21.15.78:6816/2186157269 1 ==== mgr_command_reply(tid 0: 0 instructed osd(s) 0 to deep-scrub) ==== 41+0+0 (unknown 2057319476 0 0) 0x7f61d00698c0 con 0x7f61b40514b0 2024-09-21T09:22:40.649 INFO:teuthology.orchestra.run.smithi078.stderr:instructed osd(s) 0 to deep-scrub 2024-09-21T09:22:40.651 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.649+0000 7f61d843c640 1 -- 172.21.15.78:0/2167494167 >> v1:172.21.15.78:6816/2186157269 conn(0x7f61b40514b0 legacy=0x7f61b4053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:40.651 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.649+0000 7f61d843c640 1 -- 172.21.15.78:0/2167494167 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f61d0098d40 msgr2=0x7f61d01c4cb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:40.651 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.649+0000 7f61d843c640 1 --2- 172.21.15.78:0/2167494167 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f61d0098d40 0x7f61d01c4cb0 secure :-1 s=READY pgs=58 cs=0 l=1 rev1=1 crypto rx=0x7f61cc009780 tx=0x7f61cc033b70 comp rx=0 tx=0).stop 2024-09-21T09:22:40.652 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.649+0000 7f61d843c640 1 -- 172.21.15.78:0/2167494167 shutdown_connections 2024-09-21T09:22:40.652 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.649+0000 7f61d843c640 1 --2- 172.21.15.78:0/2167494167 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f61d012eff0 0x7f61d00713d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:40.652 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.649+0000 7f61d843c640 1 --2- 172.21.15.78:0/2167494167 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f61d0099660 0x7f61d01c51f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:40.652 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.649+0000 7f61d843c640 1 --2- 172.21.15.78:0/2167494167 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f61d0098d40 0x7f61d01c4cb0 unknown :-1 s=CLOSED pgs=58 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:40.652 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.649+0000 7f61d843c640 1 -- 172.21.15.78:0/2167494167 >> 172.21.15.78:0/2167494167 conn(0x7f61d00a1790 msgr2=0x7f61d009c1d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:40.652 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.649+0000 7f61d843c640 1 -- 172.21.15.78:0/2167494167 shutdown_connections 2024-09-21T09:22:40.652 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.649+0000 7f61d843c640 1 -- 172.21.15.78:0/2167494167 wait complete. 2024-09-21T09:22:40.662 INFO:tasks.ceph:Scrubbing osd.1 2024-09-21T09:22:40.662 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.1 config set osd_debug_deep_scrub_sleep 0 2024-09-21T09:22:40.765 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.763+0000 7f973caac640 1 -- 172.21.15.78:0/3474991094 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f9738096d20 msgr2=0x7f9738094de0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:40.766 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.763+0000 7f973caac640 1 --2- 172.21.15.78:0/3474991094 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f9738096d20 0x7f9738094de0 secure :-1 s=READY pgs=136 cs=0 l=1 rev1=1 crypto rx=0x7f9730005fd0 tx=0x7f973002a820 comp rx=0 tx=0).stop 2024-09-21T09:22:40.766 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.763+0000 7f973caac640 1 -- 172.21.15.78:0/3474991094 shutdown_connections 2024-09-21T09:22:40.766 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.763+0000 7f973caac640 1 --2- 172.21.15.78:0/3474991094 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f9738096d20 0x7f9738094de0 unknown :-1 s=CLOSED pgs=136 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:40.766 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.763+0000 7f973caac640 1 --2- 172.21.15.78:0/3474991094 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f973812b590 0x7f973812b970 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:40.766 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.763+0000 7f973caac640 1 -- 172.21.15.78:0/3474991094 >> 172.21.15.78:0/3474991094 conn(0x7f973809a3f0 msgr2=0x7f9738097bd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:40.766 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.763+0000 7f973caac640 1 -- 172.21.15.78:0/3474991094 shutdown_connections 2024-09-21T09:22:40.766 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.763+0000 7f973caac640 1 -- 172.21.15.78:0/3474991094 wait complete. 2024-09-21T09:22:40.768 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.764+0000 7f973caac640 1 Processor -- start 2024-09-21T09:22:40.768 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.764+0000 7f973caac640 1 -- start start 2024-09-21T09:22:40.769 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.764+0000 7f973caac640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f9738096d20 0x7f973812c540 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:40.769 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.764+0000 7f973caac640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f973812b590 0x7f973812ca80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:40.769 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.764+0000 7f973caac640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f973812beb0 0x7f973812cfc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:40.769 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.764+0000 7f973caac640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7f9738133dd0 con 0x7f973812beb0 2024-09-21T09:22:40.769 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.764+0000 7f973caac640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7f9738133c50 con 0x7f973812b590 2024-09-21T09:22:40.769 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.764+0000 7f973caac640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7f9738133f50 con 0x7f9738096d20 2024-09-21T09:22:40.769 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.765+0000 7f973659c640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f9738096d20 0x7f973812c540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:40.769 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.765+0000 7f9736d9d640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f973812beb0 0x7f973812cfc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:40.769 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.765+0000 7f972dd9b640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f973812b590 0x7f973812ca80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:40.769 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.765+0000 7f973659c640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f9738096d20 0x7f973812c540 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3302/0 says I am v2:172.21.15.78:35904/0 (socket says 172.21.15.78:35904) 2024-09-21T09:22:40.769 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.765+0000 7f973659c640 1 -- 172.21.15.78:0/1349192004 learned_addr learned my addr 172.21.15.78:0/1349192004 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:40.769 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.765+0000 7f9736d9d640 1 -- 172.21.15.78:0/1349192004 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f9738096d20 msgr2=0x7f973812c540 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:40.769 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.765+0000 7f9736d9d640 1 --2- 172.21.15.78:0/1349192004 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f9738096d20 0x7f973812c540 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:40.769 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.765+0000 7f9736d9d640 1 -- 172.21.15.78:0/1349192004 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f973812b590 msgr2=0x7f973812ca80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:40.770 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.765+0000 7f9736d9d640 1 --2- 172.21.15.78:0/1349192004 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f973812b590 0x7f973812ca80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:40.770 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.765+0000 7f9736d9d640 1 -- 172.21.15.78:0/1349192004 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f9738093770 con 0x7f973812beb0 2024-09-21T09:22:40.770 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.765+0000 7f973659c640 1 --2- 172.21.15.78:0/1349192004 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f9738096d20 0x7f973812c540 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:22:40.770 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.765+0000 7f9736d9d640 1 --2- 172.21.15.78:0/1349192004 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f973812beb0 0x7f973812cfc0 secure :-1 s=READY pgs=137 cs=0 l=1 rev1=1 crypto rx=0x7f9730009000 tx=0x7f9730032c90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:40.770 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.765+0000 7f972f7fe640 1 -- 172.21.15.78:0/1349192004 <== mon.0 v2:172.21.15.78:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9730037040 con 0x7f973812beb0 2024-09-21T09:22:40.770 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.765+0000 7f972f7fe640 1 -- 172.21.15.78:0/1349192004 <== mon.0 v2:172.21.15.78:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f9730042070 con 0x7f973812beb0 2024-09-21T09:22:40.770 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.765+0000 7f973caac640 1 -- 172.21.15.78:0/1349192004 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f973812de30 con 0x7f973812beb0 2024-09-21T09:22:40.770 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.765+0000 7f972f7fe640 1 -- 172.21.15.78:0/1349192004 <== mon.0 v2:172.21.15.78:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f97300042a0 con 0x7f973812beb0 2024-09-21T09:22:40.770 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.765+0000 7f973caac640 1 -- 172.21.15.78:0/1349192004 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f973812e8a0 con 0x7f973812beb0 2024-09-21T09:22:40.770 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.766+0000 7f973caac640 1 -- 172.21.15.78:0/1349192004 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_get_version(what=osdmap handle=1) -- 0x7f9738090e60 con 0x7f973812beb0 2024-09-21T09:22:40.770 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.766+0000 7f972f7fe640 1 -- 172.21.15.78:0/1349192004 <== mon.0 v2:172.21.15.78:3300/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f9730004440 con 0x7f973812beb0 2024-09-21T09:22:40.771 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.767+0000 7f972f7fe640 1 -- 172.21.15.78:0/1349192004 <== mon.0 v2:172.21.15.78:3300/0 5 ==== osd_map(32..32 src has 1..32) ==== 3027+0+0 (secure 0 0 0) 0x7f9730089c10 con 0x7f973812beb0 2024-09-21T09:22:40.771 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.767+0000 7f972f7fe640 1 -- 172.21.15.78:0/1349192004 --> v1:172.21.15.78:6802/3598780309 -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7f973003ee50 con 0x7f9718057d30 2024-09-21T09:22:40.771 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.767+0000 7f972f7fe640 1 -- 172.21.15.78:0/1349192004 <== mon.0 v2:172.21.15.78:3300/0 6 ==== mon_get_version_reply(handle=1 version=32) ==== 24+0+0 (secure 0 0 0) 0x7f973008c310 con 0x7f973812beb0 2024-09-21T09:22:40.771 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.768+0000 7f972f7fe640 1 -- 172.21.15.78:0/1349192004 <== osd.1 v1:172.21.15.78:6802/3598780309 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (unknown 0 0 1302949928) 0x7f973003ee50 con 0x7f9718057d30 2024-09-21T09:22:40.782 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.780+0000 7f973caac640 1 -- 172.21.15.78:0/1349192004 --> v1:172.21.15.78:6802/3598780309 -- command(tid 2: {"prefix": "config set", "var": "osd_debug_deep_scrub_sleep", "val": ["0"]}) -- 0x7f97380741c0 con 0x7f9718057d30 2024-09-21T09:22:40.782 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.780+0000 7f972f7fe640 1 -- 172.21.15.78:0/1349192004 <== osd.1 v1:172.21.15.78:6802/3598780309 2 ==== command_reply(tid 2: 0 ) ==== 8+0+565 (unknown 0 0 125856537) 0x7f97380741c0 con 0x7f9718057d30 2024-09-21T09:22:40.783 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-21T09:22:40.783 INFO:teuthology.orchestra.run.smithi078.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' " 2024-09-21T09:22:40.783 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-21T09:22:40.784 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.780+0000 7f973caac640 1 -- 172.21.15.78:0/1349192004 >> v1:172.21.15.78:6802/3598780309 conn(0x7f9718057d30 legacy=0x7f971805a170 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:40.784 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.781+0000 7f973caac640 1 -- 172.21.15.78:0/1349192004 >> v1:172.21.15.78:6816/2186157269 conn(0x7f97180514b0 legacy=0x7f9718053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:40.784 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.781+0000 7f973caac640 1 -- 172.21.15.78:0/1349192004 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f973812beb0 msgr2=0x7f973812cfc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:40.784 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.781+0000 7f973caac640 1 --2- 172.21.15.78:0/1349192004 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f973812beb0 0x7f973812cfc0 secure :-1 s=READY pgs=137 cs=0 l=1 rev1=1 crypto rx=0x7f9730009000 tx=0x7f9730032c90 comp rx=0 tx=0).stop 2024-09-21T09:22:40.784 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.781+0000 7f9736d9d640 1 -- 172.21.15.78:0/1349192004 reap_dead start 2024-09-21T09:22:40.784 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.781+0000 7f973caac640 1 -- 172.21.15.78:0/1349192004 shutdown_connections 2024-09-21T09:22:40.784 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.781+0000 7f973caac640 1 -- 172.21.15.78:0/1349192004 >> 172.21.15.78:0/1349192004 conn(0x7f973809a3f0 msgr2=0x7f97380901a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:40.784 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.781+0000 7f973caac640 1 -- 172.21.15.78:0/1349192004 shutdown_connections 2024-09-21T09:22:40.784 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.781+0000 7f973caac640 1 -- 172.21.15.78:0/1349192004 wait complete. 2024-09-21T09:22:40.793 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd deep-scrub 1 2024-09-21T09:22:40.894 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.892+0000 7f335f4c4640 1 -- 172.21.15.78:0/1624010375 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f335812eb30 msgr2=0x7f3358130f20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:40.894 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.892+0000 7f335f4c4640 1 --2- 172.21.15.78:0/1624010375 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f335812eb30 0x7f3358130f20 secure :-1 s=READY pgs=59 cs=0 l=1 rev1=1 crypto rx=0x7f3348009830 tx=0x7f334802a510 comp rx=0 tx=0).stop 2024-09-21T09:22:40.895 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.892+0000 7f335f4c4640 1 -- 172.21.15.78:0/1624010375 shutdown_connections 2024-09-21T09:22:40.895 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.892+0000 7f335f4c4640 1 --2- 172.21.15.78:0/1624010375 >> v2:172.21.15.78:3300/0 conn(0x7f3358131460 0x7f3358133850 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:40.895 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.892+0000 7f335f4c4640 1 --2- 172.21.15.78:0/1624010375 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f335812eb30 0x7f3358130f20 unknown :-1 s=CLOSED pgs=59 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:40.895 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.892+0000 7f335f4c4640 1 -- 172.21.15.78:0/1624010375 >> 172.21.15.78:0/1624010375 conn(0x7f335809afd0 msgr2=0x7f335809b3e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:40.895 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.892+0000 7f335f4c4640 1 -- 172.21.15.78:0/1624010375 shutdown_connections 2024-09-21T09:22:40.895 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.892+0000 7f335f4c4640 1 -- 172.21.15.78:0/1624010375 wait complete. 2024-09-21T09:22:40.897 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.893+0000 7f335f4c4640 1 Processor -- start 2024-09-21T09:22:40.897 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.893+0000 7f335f4c4640 1 -- start start 2024-09-21T09:22:40.897 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.893+0000 7f335f4c4640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f335812bad0 0x7f335808c1f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:40.897 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.893+0000 7f335f4c4640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f335812eb30 0x7f335808c730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:40.897 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.893+0000 7f335f4c4640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f3358131460 0x7f335808f420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:40.897 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.893+0000 7f335f4c4640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7f3358134680 con 0x7f335812bad0 2024-09-21T09:22:40.897 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.893+0000 7f335f4c4640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7f3358134500 con 0x7f335812eb30 2024-09-21T09:22:40.897 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.893+0000 7f335f4c4640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7f3358134800 con 0x7f3358131460 2024-09-21T09:22:40.897 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.893+0000 7f335d260640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f335812bad0 0x7f335808c1f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:40.898 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.893+0000 7f335da61640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f3358131460 0x7f335808f420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:40.898 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.893+0000 7f335d260640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f335812bad0 0x7f335808c1f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3300/0 says I am v2:172.21.15.78:44760/0 (socket says 172.21.15.78:44760) 2024-09-21T09:22:40.898 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.893+0000 7f335d260640 1 -- 172.21.15.78:0/3971779256 learned_addr learned my addr 172.21.15.78:0/3971779256 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:40.898 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.893+0000 7f335ca5f640 1 --2- 172.21.15.78:0/3971779256 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f335812eb30 0x7f335808c730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:40.898 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.894+0000 7f335d260640 1 -- 172.21.15.78:0/3971779256 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f3358131460 msgr2=0x7f335808f420 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:40.898 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.894+0000 7f335d260640 1 --2- 172.21.15.78:0/3971779256 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f3358131460 0x7f335808f420 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:40.898 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.894+0000 7f335d260640 1 -- 172.21.15.78:0/3971779256 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f335812eb30 msgr2=0x7f335808c730 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:40.898 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.894+0000 7f335d260640 1 --2- 172.21.15.78:0/3971779256 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f335812eb30 0x7f335808c730 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:40.898 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.894+0000 7f335d260640 1 -- 172.21.15.78:0/3971779256 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f3358074950 con 0x7f335812bad0 2024-09-21T09:22:40.898 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.894+0000 7f335da61640 1 --2- 172.21.15.78:0/3971779256 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f3358131460 0x7f335808f420 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:22:40.898 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.894+0000 7f335d260640 1 --2- 172.21.15.78:0/3971779256 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f335812bad0 0x7f335808c1f0 secure :-1 s=READY pgs=139 cs=0 l=1 rev1=1 crypto rx=0x7f335400c490 tx=0x7f335400c890 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:40.899 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.894+0000 7f334dffb640 1 -- 172.21.15.78:0/3971779256 <== mon.0 v2:172.21.15.78:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3354011790 con 0x7f335812bad0 2024-09-21T09:22:40.899 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.894+0000 7f334dffb640 1 -- 172.21.15.78:0/3971779256 <== mon.0 v2:172.21.15.78:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f3354017070 con 0x7f335812bad0 2024-09-21T09:22:40.899 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.894+0000 7f335f4c4640 1 -- 172.21.15.78:0/3971779256 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f335808dd00 con 0x7f335812bad0 2024-09-21T09:22:40.899 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.894+0000 7f334dffb640 1 -- 172.21.15.78:0/3971779256 <== mon.0 v2:172.21.15.78:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3354011ce0 con 0x7f335812bad0 2024-09-21T09:22:40.899 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.894+0000 7f335f4c4640 1 -- 172.21.15.78:0/3971779256 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f335808e6b0 con 0x7f335812bad0 2024-09-21T09:22:40.899 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.895+0000 7f335f4c4640 1 -- 172.21.15.78:0/3971779256 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f335808e8e0 con 0x7f335812bad0 2024-09-21T09:22:40.899 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.895+0000 7f334dffb640 1 -- 172.21.15.78:0/3971779256 <== mon.0 v2:172.21.15.78:3300/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f335401c460 con 0x7f335812bad0 2024-09-21T09:22:40.900 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.896+0000 7f334dffb640 1 -- 172.21.15.78:0/3971779256 <== mon.0 v2:172.21.15.78:3300/0 5 ==== osd_map(32..32 src has 1..32) ==== 3027+0+0 (secure 0 0 0) 0x7f3354068cc0 con 0x7f335812bad0 2024-09-21T09:22:40.901 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:40.899+0000 7f334dffb640 1 -- 172.21.15.78:0/3971779256 <== mon.0 v2:172.21.15.78:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f3354067d70 con 0x7f335812bad0 2024-09-21T09:22:41.049 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.047+0000 7f335f4c4640 1 -- 172.21.15.78:0/3971779256 --> v1:172.21.15.78:6816/2186157269 -- mgr_command(tid 0: {"prefix": "osd deep-scrub", "who": "1", "target": ["mon-mgr", ""]}) -- 0x7f335812beb0 con 0x7f333c0514b0 2024-09-21T09:22:41.049 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.047+0000 7f334dffb640 1 -- 172.21.15.78:0/3971779256 <== mgr.14103 v1:172.21.15.78:6816/2186157269 1 ==== mgr_command_reply(tid 0: 0 instructed osd(s) 1 to deep-scrub) ==== 41+0+0 (unknown 489605429 0 0) 0x7f335812beb0 con 0x7f333c0514b0 2024-09-21T09:22:41.049 INFO:teuthology.orchestra.run.smithi078.stderr:instructed osd(s) 1 to deep-scrub 2024-09-21T09:22:41.051 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.049+0000 7f335f4c4640 1 -- 172.21.15.78:0/3971779256 >> v1:172.21.15.78:6816/2186157269 conn(0x7f333c0514b0 legacy=0x7f333c053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:41.051 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.049+0000 7f335f4c4640 1 -- 172.21.15.78:0/3971779256 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f335812bad0 msgr2=0x7f335808c1f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:41.051 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.049+0000 7f335f4c4640 1 --2- 172.21.15.78:0/3971779256 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f335812bad0 0x7f335808c1f0 secure :-1 s=READY pgs=139 cs=0 l=1 rev1=1 crypto rx=0x7f335400c490 tx=0x7f335400c890 comp rx=0 tx=0).stop 2024-09-21T09:22:41.052 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.049+0000 7f335f4c4640 1 -- 172.21.15.78:0/3971779256 shutdown_connections 2024-09-21T09:22:41.052 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.049+0000 7f335f4c4640 1 --2- 172.21.15.78:0/3971779256 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f3358131460 0x7f335808f420 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:41.052 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.049+0000 7f335f4c4640 1 --2- 172.21.15.78:0/3971779256 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f335812eb30 0x7f335808c730 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:41.052 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.049+0000 7f335f4c4640 1 --2- 172.21.15.78:0/3971779256 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f335812bad0 0x7f335808c1f0 unknown :-1 s=CLOSED pgs=139 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:41.052 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.049+0000 7f335f4c4640 1 -- 172.21.15.78:0/3971779256 >> 172.21.15.78:0/3971779256 conn(0x7f335809afd0 msgr2=0x7f3358099390 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:41.052 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.049+0000 7f335f4c4640 1 -- 172.21.15.78:0/3971779256 shutdown_connections 2024-09-21T09:22:41.052 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.049+0000 7f335f4c4640 1 -- 172.21.15.78:0/3971779256 wait complete. 2024-09-21T09:22:41.062 INFO:tasks.ceph:Scrubbing osd.2 2024-09-21T09:22:41.062 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.2 config set osd_debug_deep_scrub_sleep 0 2024-09-21T09:22:41.165 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.162+0000 7f0608dee640 1 -- 172.21.15.78:0/1252511821 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f0604095030 msgr2=0x7f0604133bb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:41.165 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.162+0000 7f0608dee640 1 --2- 172.21.15.78:0/1252511821 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f0604095030 0x7f0604133bb0 secure :-1 s=READY pgs=140 cs=0 l=1 rev1=1 crypto rx=0x7f05fc005fd0 tx=0x7f05fc02a8e0 comp rx=0 tx=0).stop 2024-09-21T09:22:41.165 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.162+0000 7f0608dee640 1 -- 172.21.15.78:0/1252511821 shutdown_connections 2024-09-21T09:22:41.165 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.162+0000 7f0608dee640 1 --2- 172.21.15.78:0/1252511821 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f06040946d0 0x7f0604094af0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:41.165 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.162+0000 7f0608dee640 1 --2- 172.21.15.78:0/1252511821 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f0604095030 0x7f0604133bb0 unknown :-1 s=CLOSED pgs=140 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:41.165 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.162+0000 7f0608dee640 1 --2- 172.21.15.78:0/1252511821 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f060412cd10 0x7f060412d0f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:41.165 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.162+0000 7f0608dee640 1 -- 172.21.15.78:0/1252511821 >> 172.21.15.78:0/1252511821 conn(0x7f060409a3f0 msgr2=0x7f0604097bd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:41.166 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.162+0000 7f0608dee640 1 -- 172.21.15.78:0/1252511821 shutdown_connections 2024-09-21T09:22:41.166 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.162+0000 7f0608dee640 1 -- 172.21.15.78:0/1252511821 wait complete. 2024-09-21T09:22:41.168 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.163+0000 7f0608dee640 1 Processor -- start 2024-09-21T09:22:41.168 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.163+0000 7f0608dee640 1 -- start start 2024-09-21T09:22:41.168 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.164+0000 7f0608dee640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f06040946d0 0x7f0604130830 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:41.168 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.164+0000 7f0608dee640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f0604095030 0x7f0604130d70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:41.168 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.164+0000 7f0608dee640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f060412cd10 0x7f06041312b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:41.168 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.164+0000 7f0608dee640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7f0604134610 con 0x7f06040946d0 2024-09-21T09:22:41.168 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.164+0000 7f0608dee640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7f0604134490 con 0x7f0604095030 2024-09-21T09:22:41.168 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.164+0000 7f0608dee640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7f0604134790 con 0x7f060412cd10 2024-09-21T09:22:41.168 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.164+0000 7f060259c640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f06040946d0 0x7f0604130830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:41.168 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.164+0000 7f0601d9b640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f0604095030 0x7f0604130d70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:41.169 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.164+0000 7f060259c640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f06040946d0 0x7f0604130830 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3300/0 says I am v2:172.21.15.78:44782/0 (socket says 172.21.15.78:44782) 2024-09-21T09:22:41.169 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.164+0000 7f060259c640 1 -- 172.21.15.78:0/1780184308 learned_addr learned my addr 172.21.15.78:0/1780184308 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:41.169 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.164+0000 7f0602d9d640 1 --2- 172.21.15.78:0/1780184308 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f060412cd10 0x7f06041312b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:41.169 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.164+0000 7f060259c640 1 -- 172.21.15.78:0/1780184308 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f060412cd10 msgr2=0x7f06041312b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:41.169 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.164+0000 7f060259c640 1 --2- 172.21.15.78:0/1780184308 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f060412cd10 0x7f06041312b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:41.169 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.164+0000 7f060259c640 1 -- 172.21.15.78:0/1780184308 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f0604095030 msgr2=0x7f0604130d70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:41.169 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.164+0000 7f060259c640 1 --2- 172.21.15.78:0/1780184308 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f0604095030 0x7f0604130d70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:41.169 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.164+0000 7f060259c640 1 -- 172.21.15.78:0/1780184308 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f0604097590 con 0x7f06040946d0 2024-09-21T09:22:41.169 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.164+0000 7f0601d9b640 1 --2- 172.21.15.78:0/1780184308 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f0604095030 0x7f0604130d70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:22:41.169 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.164+0000 7f060259c640 1 --2- 172.21.15.78:0/1780184308 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f06040946d0 0x7f0604130830 secure :-1 s=READY pgs=141 cs=0 l=1 rev1=1 crypto rx=0x7f05f400ea60 tx=0x7f05f400ef20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:41.169 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.164+0000 7f05faffd640 1 -- 172.21.15.78:0/1780184308 <== mon.0 v2:172.21.15.78:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f05f400cde0 con 0x7f06040946d0 2024-09-21T09:22:41.169 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.164+0000 7f05faffd640 1 -- 172.21.15.78:0/1780184308 <== mon.0 v2:172.21.15.78:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f05f4004020 con 0x7f06040946d0 2024-09-21T09:22:41.169 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.164+0000 7f0608dee640 1 -- 172.21.15.78:0/1780184308 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f060406e900 con 0x7f06040946d0 2024-09-21T09:22:41.170 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.164+0000 7f05faffd640 1 -- 172.21.15.78:0/1780184308 <== mon.0 v2:172.21.15.78:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f05f4004450 con 0x7f06040946d0 2024-09-21T09:22:41.170 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.164+0000 7f0608dee640 1 -- 172.21.15.78:0/1780184308 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f06040911d0 con 0x7f06040946d0 2024-09-21T09:22:41.170 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.165+0000 7f0608dee640 1 -- 172.21.15.78:0/1780184308 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_get_version(what=osdmap handle=1) -- 0x7f0604094af0 con 0x7f06040946d0 2024-09-21T09:22:41.170 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.166+0000 7f05faffd640 1 -- 172.21.15.78:0/1780184308 <== mon.0 v2:172.21.15.78:3300/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f05f40035f0 con 0x7f06040946d0 2024-09-21T09:22:41.170 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.166+0000 7f05faffd640 1 -- 172.21.15.78:0/1780184308 <== mon.0 v2:172.21.15.78:3300/0 5 ==== osd_map(32..32 src has 1..32) ==== 3027+0+0 (secure 0 0 0) 0x7f05f4069880 con 0x7f06040946d0 2024-09-21T09:22:41.170 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.166+0000 7f05faffd640 1 -- 172.21.15.78:0/1780184308 --> v1:172.21.15.78:6808/2733362473 -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7f05f4007e50 con 0x7f05e4057d60 2024-09-21T09:22:41.170 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.166+0000 7f05faffd640 1 -- 172.21.15.78:0/1780184308 <== mon.0 v2:172.21.15.78:3300/0 6 ==== mon_get_version_reply(handle=1 version=32) ==== 24+0+0 (secure 0 0 0) 0x7f05f406e2c0 con 0x7f06040946d0 2024-09-21T09:22:41.171 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.167+0000 7f05faffd640 1 -- 172.21.15.78:0/1780184308 <== osd.2 v1:172.21.15.78:6808/2733362473 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (unknown 0 0 1302949928) 0x7f05f4007e50 con 0x7f05e4057d60 2024-09-21T09:22:41.181 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.179+0000 7f0608dee640 1 -- 172.21.15.78:0/1780184308 --> v1:172.21.15.78:6808/2733362473 -- command(tid 2: {"prefix": "config set", "var": "osd_debug_deep_scrub_sleep", "val": ["0"]}) -- 0x7f060409b930 con 0x7f05e4057d60 2024-09-21T09:22:41.182 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.179+0000 7f05faffd640 1 -- 172.21.15.78:0/1780184308 <== osd.2 v1:172.21.15.78:6808/2733362473 2 ==== command_reply(tid 2: 0 ) ==== 8+0+565 (unknown 0 0 125856537) 0x7f060409b930 con 0x7f05e4057d60 2024-09-21T09:22:41.182 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-21T09:22:41.182 INFO:teuthology.orchestra.run.smithi078.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' " 2024-09-21T09:22:41.182 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-21T09:22:41.183 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.180+0000 7f0608dee640 1 -- 172.21.15.78:0/1780184308 >> v1:172.21.15.78:6808/2733362473 conn(0x7f05e4057d60 legacy=0x7f05e405a1a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:41.183 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.180+0000 7f0608dee640 1 -- 172.21.15.78:0/1780184308 >> v1:172.21.15.78:6816/2186157269 conn(0x7f05e40514e0 legacy=0x7f05e40539a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:41.183 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.180+0000 7f0608dee640 1 -- 172.21.15.78:0/1780184308 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f06040946d0 msgr2=0x7f0604130830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:41.183 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.180+0000 7f0608dee640 1 --2- 172.21.15.78:0/1780184308 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f06040946d0 0x7f0604130830 secure :-1 s=READY pgs=141 cs=0 l=1 rev1=1 crypto rx=0x7f05f400ea60 tx=0x7f05f400ef20 comp rx=0 tx=0).stop 2024-09-21T09:22:41.183 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.180+0000 7f0602d9d640 1 -- 172.21.15.78:0/1780184308 reap_dead start 2024-09-21T09:22:41.183 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.180+0000 7f0608dee640 1 -- 172.21.15.78:0/1780184308 shutdown_connections 2024-09-21T09:22:41.184 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.180+0000 7f0608dee640 1 -- 172.21.15.78:0/1780184308 >> 172.21.15.78:0/1780184308 conn(0x7f060409a3f0 msgr2=0x7f060409afd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:41.184 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.180+0000 7f0608dee640 1 -- 172.21.15.78:0/1780184308 shutdown_connections 2024-09-21T09:22:41.184 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.180+0000 7f0608dee640 1 -- 172.21.15.78:0/1780184308 wait complete. 2024-09-21T09:22:41.193 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd deep-scrub 2 2024-09-21T09:22:41.294 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.291+0000 7ff83608c640 1 -- 172.21.15.78:0/2292763883 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7ff830098d40 msgr2=0x7ff830099120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:41.294 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.291+0000 7ff83608c640 1 --2- 172.21.15.78:0/2292763883 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7ff830098d40 0x7ff830099120 secure :-1 s=READY pgs=60 cs=0 l=1 rev1=1 crypto rx=0x7ff820009830 tx=0x7ff82002a360 comp rx=0 tx=0).stop 2024-09-21T09:22:41.294 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.291+0000 7ff83608c640 1 -- 172.21.15.78:0/2292763883 shutdown_connections 2024-09-21T09:22:41.294 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.291+0000 7ff83608c640 1 --2- 172.21.15.78:0/2292763883 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7ff830099660 0x7ff83012eab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:41.294 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.291+0000 7ff83608c640 1 --2- 172.21.15.78:0/2292763883 >> v2:172.21.15.78:3300/0 conn(0x7ff83012eff0 0x7ff83012f480 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:41.294 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.291+0000 7ff83608c640 1 --2- 172.21.15.78:0/2292763883 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7ff830098d40 0x7ff830099120 unknown :-1 s=CLOSED pgs=60 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:41.294 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.291+0000 7ff83608c640 1 -- 172.21.15.78:0/2292763883 >> 172.21.15.78:0/2292763883 conn(0x7ff8300a1790 msgr2=0x7ff8300a1ba0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:41.294 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.291+0000 7ff83608c640 1 -- 172.21.15.78:0/2292763883 shutdown_connections 2024-09-21T09:22:41.295 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.291+0000 7ff83608c640 1 -- 172.21.15.78:0/2292763883 wait complete. 2024-09-21T09:22:41.295 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.292+0000 7ff83608c640 1 Processor -- start 2024-09-21T09:22:41.295 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.292+0000 7ff83608c640 1 -- start start 2024-09-21T09:22:41.297 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.292+0000 7ff83608c640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7ff830098d40 0x7ff830132b80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:41.297 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.292+0000 7ff83608c640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7ff830099660 0x7ff8301330c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:41.297 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.292+0000 7ff83608c640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7ff83012eff0 0x7ff830133600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:41.297 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.292+0000 7ff83608c640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7ff830136610 con 0x7ff83012eff0 2024-09-21T09:22:41.297 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.292+0000 7ff83608c640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7ff830136490 con 0x7ff830098d40 2024-09-21T09:22:41.297 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.292+0000 7ff83608c640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7ff830136790 con 0x7ff830099660 2024-09-21T09:22:41.297 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.292+0000 7ff82f7fe640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7ff830098d40 0x7ff830132b80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:41.297 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.292+0000 7ff826ffd640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7ff830099660 0x7ff8301330c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:41.297 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.292+0000 7ff82f7fe640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7ff830098d40 0x7ff830132b80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3301/0 says I am v2:172.21.15.78:37734/0 (socket says 172.21.15.78:37734) 2024-09-21T09:22:41.297 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.292+0000 7ff82ffff640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7ff83012eff0 0x7ff830133600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:41.297 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.292+0000 7ff82f7fe640 1 -- 172.21.15.78:0/2046285537 learned_addr learned my addr 172.21.15.78:0/2046285537 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:41.298 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.293+0000 7ff82f7fe640 1 -- 172.21.15.78:0/2046285537 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7ff830099660 msgr2=0x7ff8301330c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:41.298 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.293+0000 7ff82f7fe640 1 --2- 172.21.15.78:0/2046285537 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7ff830099660 0x7ff8301330c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:41.298 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.293+0000 7ff82f7fe640 1 -- 172.21.15.78:0/2046285537 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7ff83012eff0 msgr2=0x7ff830133600 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:41.298 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.293+0000 7ff82f7fe640 1 --2- 172.21.15.78:0/2046285537 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7ff83012eff0 0x7ff830133600 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:41.298 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.293+0000 7ff82f7fe640 1 -- 172.21.15.78:0/2046285537 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ff830134510 con 0x7ff830098d40 2024-09-21T09:22:41.298 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.293+0000 7ff82ffff640 1 --2- 172.21.15.78:0/2046285537 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7ff83012eff0 0x7ff830133600 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:22:41.298 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.293+0000 7ff826ffd640 1 --2- 172.21.15.78:0/2046285537 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7ff830099660 0x7ff8301330c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:22:41.298 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.293+0000 7ff82f7fe640 1 --2- 172.21.15.78:0/2046285537 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7ff830098d40 0x7ff830132b80 secure :-1 s=READY pgs=61 cs=0 l=1 rev1=1 crypto rx=0x7ff820009950 tx=0x7ff820033b70 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:41.298 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.293+0000 7ff82cff9640 1 -- 172.21.15.78:0/2046285537 <== mon.1 v2:172.21.15.78:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff82003f020 con 0x7ff830098d40 2024-09-21T09:22:41.298 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.293+0000 7ff82cff9640 1 -- 172.21.15.78:0/2046285537 <== mon.1 v2:172.21.15.78:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7ff8200044c0 con 0x7ff830098d40 2024-09-21T09:22:41.298 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.293+0000 7ff83608c640 1 -- 172.21.15.78:0/2046285537 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7ff830135870 con 0x7ff830098d40 2024-09-21T09:22:41.298 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.293+0000 7ff82cff9640 1 -- 172.21.15.78:0/2046285537 <== mon.1 v2:172.21.15.78:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff820037040 con 0x7ff830098d40 2024-09-21T09:22:41.298 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.293+0000 7ff83608c640 1 -- 172.21.15.78:0/2046285537 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7ff8301c4ae0 con 0x7ff830098d40 2024-09-21T09:22:41.298 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.294+0000 7ff83608c640 1 -- 172.21.15.78:0/2046285537 --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7ff83009be20 con 0x7ff830098d40 2024-09-21T09:22:41.299 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.294+0000 7ff82cff9640 1 -- 172.21.15.78:0/2046285537 <== mon.1 v2:172.21.15.78:3301/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7ff820042020 con 0x7ff830098d40 2024-09-21T09:22:41.299 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.294+0000 7ff82cff9640 1 -- 172.21.15.78:0/2046285537 <== mon.1 v2:172.21.15.78:3301/0 5 ==== osd_map(32..32 src has 1..32) ==== 3027+0+0 (secure 0 0 0) 0x7ff820088e10 con 0x7ff830098d40 2024-09-21T09:22:41.300 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.298+0000 7ff82cff9640 1 -- 172.21.15.78:0/2046285537 <== mon.1 v2:172.21.15.78:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7ff83009be20 con 0x7ff830098d40 2024-09-21T09:22:41.449 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.447+0000 7ff83608c640 1 -- 172.21.15.78:0/2046285537 --> v1:172.21.15.78:6816/2186157269 -- mgr_command(tid 0: {"prefix": "osd deep-scrub", "who": "2", "target": ["mon-mgr", ""]}) -- 0x7ff830099120 con 0x7ff8100514b0 2024-09-21T09:22:41.449 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.447+0000 7ff82cff9640 1 -- 172.21.15.78:0/2046285537 <== mgr.14103 v1:172.21.15.78:6816/2186157269 1 ==== mgr_command_reply(tid 0: 0 instructed osd(s) 2 to deep-scrub) ==== 41+0+0 (unknown 3049115190 0 0) 0x7ff830099120 con 0x7ff8100514b0 2024-09-21T09:22:41.449 INFO:teuthology.orchestra.run.smithi078.stderr:instructed osd(s) 2 to deep-scrub 2024-09-21T09:22:41.451 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.448+0000 7ff83608c640 1 -- 172.21.15.78:0/2046285537 >> v1:172.21.15.78:6816/2186157269 conn(0x7ff8100514b0 legacy=0x7ff810053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:41.451 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.448+0000 7ff83608c640 1 -- 172.21.15.78:0/2046285537 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7ff830098d40 msgr2=0x7ff830132b80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:41.451 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.448+0000 7ff83608c640 1 --2- 172.21.15.78:0/2046285537 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7ff830098d40 0x7ff830132b80 secure :-1 s=READY pgs=61 cs=0 l=1 rev1=1 crypto rx=0x7ff820009950 tx=0x7ff820033b70 comp rx=0 tx=0).stop 2024-09-21T09:22:41.451 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.449+0000 7ff83608c640 1 -- 172.21.15.78:0/2046285537 shutdown_connections 2024-09-21T09:22:41.451 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.449+0000 7ff83608c640 1 --2- 172.21.15.78:0/2046285537 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7ff83012eff0 0x7ff830133600 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:41.451 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.449+0000 7ff83608c640 1 --2- 172.21.15.78:0/2046285537 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7ff830099660 0x7ff8301330c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:41.452 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.449+0000 7ff83608c640 1 --2- 172.21.15.78:0/2046285537 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7ff830098d40 0x7ff830132b80 unknown :-1 s=CLOSED pgs=61 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:41.452 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.449+0000 7ff83608c640 1 -- 172.21.15.78:0/2046285537 >> 172.21.15.78:0/2046285537 conn(0x7ff8300a1790 msgr2=0x7ff830128440 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:41.452 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.449+0000 7ff83608c640 1 -- 172.21.15.78:0/2046285537 shutdown_connections 2024-09-21T09:22:41.452 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.449+0000 7ff83608c640 1 -- 172.21.15.78:0/2046285537 wait complete. 2024-09-21T09:22:41.462 INFO:tasks.ceph:Scrubbing osd.3 2024-09-21T09:22:41.462 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.3 config set osd_debug_deep_scrub_sleep 0 2024-09-21T09:22:41.567 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.565+0000 7f7ebd21d640 1 -- 172.21.15.78:0/2501603213 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f7eb8096330 msgr2=0x7f7eb8133ad0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:41.567 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.565+0000 7f7ebd21d640 1 --2- 172.21.15.78:0/2501603213 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f7eb8096330 0x7f7eb8133ad0 secure :-1 s=READY pgs=60 cs=0 l=1 rev1=1 crypto rx=0x7f7eb0009ca0 tx=0x7f7eb002a9a0 comp rx=0 tx=0).stop 2024-09-21T09:22:41.567 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.565+0000 7f7ebd21d640 1 -- 172.21.15.78:0/2501603213 shutdown_connections 2024-09-21T09:22:41.567 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.565+0000 7f7ebd21d640 1 --2- 172.21.15.78:0/2501603213 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f7eb8096330 0x7f7eb8133ad0 unknown :-1 s=CLOSED pgs=60 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:41.568 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.565+0000 7f7ebd21d640 1 --2- 172.21.15.78:0/2501603213 >> v2:172.21.15.78:3300/0 conn(0x7f7eb812bc10 0x7f7eb812bff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:41.568 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.565+0000 7f7ebd21d640 1 -- 172.21.15.78:0/2501603213 >> 172.21.15.78:0/2501603213 conn(0x7f7eb809a3f0 msgr2=0x7f7eb8097bd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:41.568 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.565+0000 7f7ebd21d640 1 -- 172.21.15.78:0/2501603213 shutdown_connections 2024-09-21T09:22:41.568 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.565+0000 7f7ebd21d640 1 -- 172.21.15.78:0/2501603213 wait complete. 2024-09-21T09:22:41.570 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.566+0000 7f7ebd21d640 1 Processor -- start 2024-09-21T09:22:41.570 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.566+0000 7f7ebd21d640 1 -- start start 2024-09-21T09:22:41.570 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.566+0000 7f7ebd21d640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f7eb80957e0 0x7f7eb8073d50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:41.570 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.566+0000 7f7ebd21d640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f7eb8096330 0x7f7eb8074290 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:41.570 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.566+0000 7f7ebd21d640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f7eb812bc10 0x7f7eb808bfb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:41.570 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.566+0000 7f7ebd21d640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7f7eb8134750 con 0x7f7eb812bc10 2024-09-21T09:22:41.570 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.566+0000 7f7ebd21d640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7f7eb81345d0 con 0x7f7eb80957e0 2024-09-21T09:22:41.570 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.566+0000 7f7ebd21d640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7f7eb81348d0 con 0x7f7eb8096330 2024-09-21T09:22:41.570 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.566+0000 7f7eb659c640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f7eb8096330 0x7f7eb8074290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:41.571 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.566+0000 7f7eb759e640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f7eb812bc10 0x7f7eb808bfb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:41.571 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.566+0000 7f7eb6d9d640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f7eb80957e0 0x7f7eb8073d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:41.571 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.566+0000 7f7eb659c640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f7eb8096330 0x7f7eb8074290 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3302/0 says I am v2:172.21.15.78:35980/0 (socket says 172.21.15.78:35980) 2024-09-21T09:22:41.571 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.566+0000 7f7eb659c640 1 -- 172.21.15.78:0/2591888458 learned_addr learned my addr 172.21.15.78:0/2591888458 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:41.571 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.567+0000 7f7eb759e640 1 -- 172.21.15.78:0/2591888458 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f7eb8096330 msgr2=0x7f7eb8074290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:41.571 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.567+0000 7f7eb759e640 1 --2- 172.21.15.78:0/2591888458 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f7eb8096330 0x7f7eb8074290 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:41.571 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.567+0000 7f7eb759e640 1 -- 172.21.15.78:0/2591888458 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f7eb80957e0 msgr2=0x7f7eb8073d50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:41.571 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.567+0000 7f7eb759e640 1 --2- 172.21.15.78:0/2591888458 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f7eb80957e0 0x7f7eb8073d50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:41.571 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.567+0000 7f7eb759e640 1 -- 172.21.15.78:0/2591888458 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f7eb8093fa0 con 0x7f7eb812bc10 2024-09-21T09:22:41.571 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.567+0000 7f7eb759e640 1 --2- 172.21.15.78:0/2591888458 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f7eb812bc10 0x7f7eb808bfb0 secure :-1 s=READY pgs=143 cs=0 l=1 rev1=1 crypto rx=0x7f7eb0032000 tx=0x7f7eb0031e90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:41.571 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.567+0000 7f7e9f7fe640 1 -- 172.21.15.78:0/2591888458 <== mon.0 v2:172.21.15.78:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7eb0047070 con 0x7f7eb812bc10 2024-09-21T09:22:41.571 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.567+0000 7f7e9f7fe640 1 -- 172.21.15.78:0/2591888458 <== mon.0 v2:172.21.15.78:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f7eb0004250 con 0x7f7eb812bc10 2024-09-21T09:22:41.571 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.567+0000 7f7ebd21d640 1 -- 172.21.15.78:0/2591888458 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f7eb8126780 con 0x7f7eb812bc10 2024-09-21T09:22:41.571 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.567+0000 7f7e9f7fe640 1 -- 172.21.15.78:0/2591888458 <== mon.0 v2:172.21.15.78:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7eb0034070 con 0x7f7eb812bc10 2024-09-21T09:22:41.572 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.567+0000 7f7ebd21d640 1 -- 172.21.15.78:0/2591888458 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f7eb808c4f0 con 0x7f7eb812bc10 2024-09-21T09:22:41.572 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.568+0000 7f7ebd21d640 1 -- 172.21.15.78:0/2591888458 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_get_version(what=osdmap handle=1) -- 0x7f7eb809c9e0 con 0x7f7eb812bc10 2024-09-21T09:22:41.572 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.568+0000 7f7e9f7fe640 1 -- 172.21.15.78:0/2591888458 <== mon.0 v2:172.21.15.78:3300/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f7eb004a070 con 0x7f7eb812bc10 2024-09-21T09:22:41.572 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.569+0000 7f7e9f7fe640 1 -- 172.21.15.78:0/2591888458 <== mon.0 v2:172.21.15.78:3300/0 5 ==== osd_map(32..32 src has 1..32) ==== 3027+0+0 (secure 0 0 0) 0x7f7eb0089c20 con 0x7f7eb812bc10 2024-09-21T09:22:41.573 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.569+0000 7f7e9f7fe640 1 -- 172.21.15.78:0/2591888458 --> v1:172.21.15.78:6812/3499964365 -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7f7eb002eec0 con 0x7f7ea4057da0 2024-09-21T09:22:41.573 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.569+0000 7f7e9f7fe640 1 -- 172.21.15.78:0/2591888458 <== mon.0 v2:172.21.15.78:3300/0 6 ==== mon_get_version_reply(handle=1 version=32) ==== 24+0+0 (secure 0 0 0) 0x7f7eb008c320 con 0x7f7eb812bc10 2024-09-21T09:22:41.573 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.569+0000 7f7e9f7fe640 1 -- 172.21.15.78:0/2591888458 <== osd.3 v1:172.21.15.78:6812/3499964365 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (unknown 0 0 1302949928) 0x7f7eb002eec0 con 0x7f7ea4057da0 2024-09-21T09:22:41.584 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.582+0000 7f7ebd21d640 1 -- 172.21.15.78:0/2591888458 --> v1:172.21.15.78:6812/3499964365 -- command(tid 2: {"prefix": "config set", "var": "osd_debug_deep_scrub_sleep", "val": ["0"]}) -- 0x7f7eb8093000 con 0x7f7ea4057da0 2024-09-21T09:22:41.584 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.582+0000 7f7e9f7fe640 1 -- 172.21.15.78:0/2591888458 <== osd.3 v1:172.21.15.78:6812/3499964365 2 ==== command_reply(tid 2: 0 ) ==== 8+0+565 (unknown 0 0 125856537) 0x7f7eb8093000 con 0x7f7ea4057da0 2024-09-21T09:22:41.584 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-21T09:22:41.585 INFO:teuthology.orchestra.run.smithi078.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' " 2024-09-21T09:22:41.585 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-21T09:22:41.586 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.582+0000 7f7ebd21d640 1 -- 172.21.15.78:0/2591888458 >> v1:172.21.15.78:6812/3499964365 conn(0x7f7ea4057da0 legacy=0x7f7ea405a1e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:41.586 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.582+0000 7f7ebd21d640 1 -- 172.21.15.78:0/2591888458 >> v1:172.21.15.78:6816/2186157269 conn(0x7f7ea40514b0 legacy=0x7f7ea4053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:41.586 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.582+0000 7f7ebd21d640 1 -- 172.21.15.78:0/2591888458 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f7eb812bc10 msgr2=0x7f7eb808bfb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:41.586 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.582+0000 7f7ebd21d640 1 --2- 172.21.15.78:0/2591888458 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f7eb812bc10 0x7f7eb808bfb0 secure :-1 s=READY pgs=143 cs=0 l=1 rev1=1 crypto rx=0x7f7eb0032000 tx=0x7f7eb0031e90 comp rx=0 tx=0).stop 2024-09-21T09:22:41.586 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.582+0000 7f7eb759e640 1 -- 172.21.15.78:0/2591888458 reap_dead start 2024-09-21T09:22:41.586 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.582+0000 7f7ebd21d640 1 -- 172.21.15.78:0/2591888458 shutdown_connections 2024-09-21T09:22:41.586 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.582+0000 7f7ebd21d640 1 -- 172.21.15.78:0/2591888458 >> 172.21.15.78:0/2591888458 conn(0x7f7eb809a3f0 msgr2=0x7f7eb80926a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:41.586 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.583+0000 7f7ebd21d640 1 -- 172.21.15.78:0/2591888458 shutdown_connections 2024-09-21T09:22:41.586 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.583+0000 7f7ebd21d640 1 -- 172.21.15.78:0/2591888458 wait complete. 2024-09-21T09:22:41.595 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd deep-scrub 3 2024-09-21T09:22:41.695 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.692+0000 7fd4a58d7640 1 -- 172.21.15.78:0/1553997832 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fd4a012eff0 msgr2=0x7fd4a012f480 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:41.696 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.692+0000 7fd4a58d7640 1 --2- 172.21.15.78:0/1553997832 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fd4a012eff0 0x7fd4a012f480 secure :-1 s=READY pgs=144 cs=0 l=1 rev1=1 crypto rx=0x7fd498009830 tx=0x7fd49802a360 comp rx=0 tx=0).stop 2024-09-21T09:22:41.696 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.692+0000 7fd4a58d7640 1 -- 172.21.15.78:0/1553997832 shutdown_connections 2024-09-21T09:22:41.696 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.692+0000 7fd4a58d7640 1 --2- 172.21.15.78:0/1553997832 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fd4a0099660 0x7fd4a012eab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:41.696 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.692+0000 7fd4a58d7640 1 --2- 172.21.15.78:0/1553997832 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fd4a012eff0 0x7fd4a012f480 unknown :-1 s=CLOSED pgs=144 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:41.696 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.692+0000 7fd4a58d7640 1 --2- 172.21.15.78:0/1553997832 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fd4a0098d40 0x7fd4a0099120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:41.696 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.692+0000 7fd4a58d7640 1 -- 172.21.15.78:0/1553997832 >> 172.21.15.78:0/1553997832 conn(0x7fd4a00a1790 msgr2=0x7fd4a00a1ba0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:41.696 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.692+0000 7fd4a58d7640 1 -- 172.21.15.78:0/1553997832 shutdown_connections 2024-09-21T09:22:41.696 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.692+0000 7fd4a58d7640 1 -- 172.21.15.78:0/1553997832 wait complete. 2024-09-21T09:22:41.696 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.693+0000 7fd4a58d7640 1 Processor -- start 2024-09-21T09:22:41.696 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.693+0000 7fd4a58d7640 1 -- start start 2024-09-21T09:22:41.699 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.694+0000 7fd4a58d7640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fd4a0098d40 0x7fd4a01c4ce0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:41.699 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.694+0000 7fd4a58d7640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fd4a0099660 0x7fd4a01c5220 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:41.699 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.694+0000 7fd4a58d7640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fd4a012eff0 0x7fd4a01328e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:41.699 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.694+0000 7fd4a58d7640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7fd4a0136610 con 0x7fd4a012eff0 2024-09-21T09:22:41.699 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.694+0000 7fd4a58d7640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7fd4a0136490 con 0x7fd4a0098d40 2024-09-21T09:22:41.699 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.694+0000 7fd4a58d7640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7fd4a0136790 con 0x7fd4a0099660 2024-09-21T09:22:41.699 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.694+0000 7fd49effd640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fd4a0098d40 0x7fd4a01c4ce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:41.699 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.694+0000 7fd49e7fc640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fd4a0099660 0x7fd4a01c5220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:41.700 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.694+0000 7fd49effd640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fd4a0098d40 0x7fd4a01c4ce0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3301/0 says I am v2:172.21.15.78:37756/0 (socket says 172.21.15.78:37756) 2024-09-21T09:22:41.700 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.694+0000 7fd49effd640 1 -- 172.21.15.78:0/3797212270 learned_addr learned my addr 172.21.15.78:0/3797212270 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:41.700 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.694+0000 7fd49f7fe640 1 --2- 172.21.15.78:0/3797212270 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fd4a012eff0 0x7fd4a01328e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:41.700 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.694+0000 7fd49f7fe640 1 -- 172.21.15.78:0/3797212270 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fd4a0099660 msgr2=0x7fd4a01c5220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:41.700 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.694+0000 7fd49f7fe640 1 --2- 172.21.15.78:0/3797212270 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fd4a0099660 0x7fd4a01c5220 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:41.700 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.694+0000 7fd49f7fe640 1 -- 172.21.15.78:0/3797212270 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fd4a0098d40 msgr2=0x7fd4a01c4ce0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:41.700 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.694+0000 7fd49f7fe640 1 --2- 172.21.15.78:0/3797212270 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fd4a0098d40 0x7fd4a01c4ce0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:41.700 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.694+0000 7fd49f7fe640 1 -- 172.21.15.78:0/3797212270 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fd4a006e910 con 0x7fd4a012eff0 2024-09-21T09:22:41.700 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.694+0000 7fd49effd640 1 --2- 172.21.15.78:0/3797212270 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fd4a0098d40 0x7fd4a01c4ce0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:22:41.700 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.694+0000 7fd49f7fe640 1 --2- 172.21.15.78:0/3797212270 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fd4a012eff0 0x7fd4a01328e0 secure :-1 s=READY pgs=145 cs=0 l=1 rev1=1 crypto rx=0x7fd498031000 tx=0x7fd498033af0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:41.701 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.694+0000 7fd487fff640 1 -- 172.21.15.78:0/3797212270 <== mon.0 v2:172.21.15.78:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd49803f020 con 0x7fd4a012eff0 2024-09-21T09:22:41.701 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.694+0000 7fd487fff640 1 -- 172.21.15.78:0/3797212270 <== mon.0 v2:172.21.15.78:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fd498033ce0 con 0x7fd4a012eff0 2024-09-21T09:22:41.701 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.694+0000 7fd4a58d7640 1 -- 172.21.15.78:0/3797212270 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fd4a006e600 con 0x7fd4a012eff0 2024-09-21T09:22:41.701 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.694+0000 7fd487fff640 1 -- 172.21.15.78:0/3797212270 <== mon.0 v2:172.21.15.78:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd498037040 con 0x7fd4a012eff0 2024-09-21T09:22:41.701 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.695+0000 7fd4a58d7640 1 -- 172.21.15.78:0/3797212270 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fd4a01c5760 con 0x7fd4a012eff0 2024-09-21T09:22:41.701 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.696+0000 7fd4a58d7640 1 -- 172.21.15.78:0/3797212270 --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fd4a012eab0 con 0x7fd4a012eff0 2024-09-21T09:22:41.701 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.696+0000 7fd487fff640 1 -- 172.21.15.78:0/3797212270 <== mon.0 v2:172.21.15.78:3300/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7fd498042020 con 0x7fd4a012eff0 2024-09-21T09:22:41.701 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.696+0000 7fd487fff640 1 -- 172.21.15.78:0/3797212270 <== mon.0 v2:172.21.15.78:3300/0 5 ==== osd_map(32..32 src has 1..32) ==== 3027+0+0 (secure 0 0 0) 0x7fd498088d50 con 0x7fd4a012eff0 2024-09-21T09:22:41.702 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.700+0000 7fd487fff640 1 -- 172.21.15.78:0/3797212270 <== mon.0 v2:172.21.15.78:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7fd49808c030 con 0x7fd4a012eff0 2024-09-21T09:22:41.849 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.847+0000 7fd4a58d7640 1 -- 172.21.15.78:0/3797212270 --> v1:172.21.15.78:6816/2186157269 -- mgr_command(tid 0: {"prefix": "osd deep-scrub", "who": "3", "target": ["mon-mgr", ""]}) -- 0x7fd4a00698c0 con 0x7fd48c0514b0 2024-09-21T09:22:41.850 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.847+0000 7fd487fff640 1 -- 172.21.15.78:0/3797212270 <== mgr.14103 v1:172.21.15.78:6816/2186157269 1 ==== mgr_command_reply(tid 0: 0 instructed osd(s) 3 to deep-scrub) ==== 41+0+0 (unknown 3526570807 0 0) 0x7fd4a00698c0 con 0x7fd48c0514b0 2024-09-21T09:22:41.850 INFO:teuthology.orchestra.run.smithi078.stderr:instructed osd(s) 3 to deep-scrub 2024-09-21T09:22:41.851 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.849+0000 7fd4a58d7640 1 -- 172.21.15.78:0/3797212270 >> v1:172.21.15.78:6816/2186157269 conn(0x7fd48c0514b0 legacy=0x7fd48c053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:41.851 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.849+0000 7fd4a58d7640 1 -- 172.21.15.78:0/3797212270 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fd4a012eff0 msgr2=0x7fd4a01328e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:41.851 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.849+0000 7fd4a58d7640 1 --2- 172.21.15.78:0/3797212270 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fd4a012eff0 0x7fd4a01328e0 secure :-1 s=READY pgs=145 cs=0 l=1 rev1=1 crypto rx=0x7fd498031000 tx=0x7fd498033af0 comp rx=0 tx=0).stop 2024-09-21T09:22:41.852 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.849+0000 7fd4a58d7640 1 -- 172.21.15.78:0/3797212270 shutdown_connections 2024-09-21T09:22:41.852 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.849+0000 7fd4a58d7640 1 --2- 172.21.15.78:0/3797212270 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fd4a012eff0 0x7fd4a01328e0 unknown :-1 s=CLOSED pgs=145 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:41.852 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.849+0000 7fd4a58d7640 1 --2- 172.21.15.78:0/3797212270 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fd4a0099660 0x7fd4a01c5220 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:41.852 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.849+0000 7fd4a58d7640 1 --2- 172.21.15.78:0/3797212270 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fd4a0098d40 0x7fd4a01c4ce0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:41.852 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.849+0000 7fd4a58d7640 1 -- 172.21.15.78:0/3797212270 >> 172.21.15.78:0/3797212270 conn(0x7fd4a00a1790 msgr2=0x7fd4a0128440 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:41.852 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.849+0000 7fd4a58d7640 1 -- 172.21.15.78:0/3797212270 shutdown_connections 2024-09-21T09:22:41.852 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.849+0000 7fd4a58d7640 1 -- 172.21.15.78:0/3797212270 wait complete. 2024-09-21T09:22:41.862 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-09-21T09:22:41.963 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.960+0000 7f22f61bb640 1 -- 172.21.15.78:0/2214912477 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f22f00991f0 msgr2=0x7f22f012bc20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:41.963 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.960+0000 7f22f61bb640 1 --2- 172.21.15.78:0/2214912477 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f22f00991f0 0x7f22f012bc20 secure :-1 s=READY pgs=62 cs=0 l=1 rev1=1 crypto rx=0x7f22e0005fd0 tx=0x7f22e002a8e0 comp rx=0 tx=0).stop 2024-09-21T09:22:41.963 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.960+0000 7f22f61bb640 1 -- 172.21.15.78:0/2214912477 shutdown_connections 2024-09-21T09:22:41.963 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.960+0000 7f22f61bb640 1 --2- 172.21.15.78:0/2214912477 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f22f012c160 0x7f22f00956e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:41.963 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.960+0000 7f22f61bb640 1 --2- 172.21.15.78:0/2214912477 >> v2:172.21.15.78:3300/0 conn(0x7f22f0095c20 0x7f22f0096060 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:41.964 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.960+0000 7f22f61bb640 1 --2- 172.21.15.78:0/2214912477 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f22f00991f0 0x7f22f012bc20 unknown :-1 s=CLOSED pgs=62 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:41.964 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.960+0000 7f22f61bb640 1 -- 172.21.15.78:0/2214912477 >> 172.21.15.78:0/2214912477 conn(0x7f22f009a2e0 msgr2=0x7f22f0097ad0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:41.964 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.960+0000 7f22f61bb640 1 -- 172.21.15.78:0/2214912477 shutdown_connections 2024-09-21T09:22:41.964 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.960+0000 7f22f61bb640 1 -- 172.21.15.78:0/2214912477 wait complete. 2024-09-21T09:22:41.965 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.961+0000 7f22f61bb640 1 Processor -- start 2024-09-21T09:22:41.966 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.961+0000 7f22f61bb640 1 -- start start 2024-09-21T09:22:41.966 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.962+0000 7f22f61bb640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f22f0095c20 0x7f22f0072a40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:41.966 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.962+0000 7f22f61bb640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f22f00991f0 0x7f22f0072f80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:41.966 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.962+0000 7f22f61bb640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f22f012c160 0x7f22f00734c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:22:41.966 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.962+0000 7f22f61bb640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7f22f0133490 con 0x7f22f012c160 2024-09-21T09:22:41.966 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.962+0000 7f22f61bb640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7f22f0133310 con 0x7f22f00991f0 2024-09-21T09:22:41.966 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.962+0000 7f22f61bb640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7f22f0133610 con 0x7f22f0095c20 2024-09-21T09:22:41.966 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.962+0000 7f22ef7fe640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f22f0095c20 0x7f22f0072a40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:41.966 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.962+0000 7f22ef7fe640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f22f0095c20 0x7f22f0072a40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3302/0 says I am v2:172.21.15.78:36004/0 (socket says 172.21.15.78:36004) 2024-09-21T09:22:41.966 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.962+0000 7f22ef7fe640 1 -- 172.21.15.78:0/1086037420 learned_addr learned my addr 172.21.15.78:0/1086037420 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:22:41.966 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.962+0000 7f22effff640 1 --2- 172.21.15.78:0/1086037420 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f22f012c160 0x7f22f00734c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:41.966 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.962+0000 7f22eeffd640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f22f00991f0 0x7f22f0072f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:22:41.967 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.962+0000 7f22ef7fe640 1 -- 172.21.15.78:0/1086037420 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f22f00991f0 msgr2=0x7f22f0072f80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:41.967 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.962+0000 7f22ef7fe640 1 --2- 172.21.15.78:0/1086037420 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f22f00991f0 0x7f22f0072f80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:41.967 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.962+0000 7f22ef7fe640 1 -- 172.21.15.78:0/1086037420 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f22f012c160 msgr2=0x7f22f00734c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:41.967 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.962+0000 7f22ef7fe640 1 --2- 172.21.15.78:0/1086037420 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f22f012c160 0x7f22f00734c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:41.967 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.962+0000 7f22ef7fe640 1 -- 172.21.15.78:0/1086037420 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f22f0094840 con 0x7f22f0095c20 2024-09-21T09:22:41.967 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.962+0000 7f22effff640 1 --2- 172.21.15.78:0/1086037420 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f22f012c160 0x7f22f00734c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:22:41.967 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.962+0000 7f22ef7fe640 1 --2- 172.21.15.78:0/1086037420 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f22f0095c20 0x7f22f0072a40 secure :-1 s=READY pgs=61 cs=0 l=1 rev1=1 crypto rx=0x7f22e0033040 tx=0x7f22e0003f90 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:22:41.967 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.962+0000 7f22d7fff640 1 -- 172.21.15.78:0/1086037420 <== mon.2 v2:172.21.15.78:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f22e0030e90 con 0x7f22f0095c20 2024-09-21T09:22:41.967 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.963+0000 7f22d7fff640 1 -- 172.21.15.78:0/1086037420 <== mon.2 v2:172.21.15.78:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f22e0004040 con 0x7f22f0095c20 2024-09-21T09:22:41.967 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.963+0000 7f22d7fff640 1 -- 172.21.15.78:0/1086037420 <== mon.2 v2:172.21.15.78:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f22e0004490 con 0x7f22f0095c20 2024-09-21T09:22:41.967 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.963+0000 7f22f61bb640 1 -- 172.21.15.78:0/1086037420 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f22f00950c0 con 0x7f22f0095c20 2024-09-21T09:22:41.967 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.963+0000 7f22f61bb640 1 -- 172.21.15.78:0/1086037420 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f22f0073ac0 con 0x7f22f0095c20 2024-09-21T09:22:41.968 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.964+0000 7f22f61bb640 1 -- 172.21.15.78:0/1086037420 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f22f012bc20 con 0x7f22f0095c20 2024-09-21T09:22:41.968 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.964+0000 7f22d7fff640 1 -- 172.21.15.78:0/1086037420 <== mon.2 v2:172.21.15.78:3302/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f22e0004630 con 0x7f22f0095c20 2024-09-21T09:22:41.968 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.964+0000 7f22d7fff640 1 -- 172.21.15.78:0/1086037420 <== mon.2 v2:172.21.15.78:3302/0 5 ==== osd_map(32..32 src has 1..32) ==== 3027+0+0 (secure 0 0 0) 0x7f22e008a580 con 0x7f22f0095c20 2024-09-21T09:22:41.970 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:41.968+0000 7f22d7fff640 1 -- 172.21.15.78:0/1086037420 <== mon.2 v2:172.21.15.78:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f22e0089630 con 0x7f22f0095c20 2024-09-21T09:22:42.115 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:42.113+0000 7f22f61bb640 1 -- 172.21.15.78:0/1086037420 --> v1:172.21.15.78:6816/2186157269 -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) -- 0x7f22f00699c0 con 0x7f22dc0514b0 2024-09-21T09:22:42.116 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:42.114+0000 7f22d7fff640 1 -- 172.21.15.78:0/1086037420 <== mgr.14103 v1:172.21.15.78:6816/2186157269 1 ==== mgr_command_reply(tid 0: 0 dumped all) ==== 18+0+31990 (unknown 2965378022 0 2217737159) 0x7f22f00699c0 con 0x7f22dc0514b0 2024-09-21T09:22:42.116 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:22:42.117 INFO:teuthology.orchestra.run.smithi078.stderr:dumped all 2024-09-21T09:22:42.118 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:42.115+0000 7f22f61bb640 1 -- 172.21.15.78:0/1086037420 >> v1:172.21.15.78:6816/2186157269 conn(0x7f22dc0514b0 legacy=0x7f22dc053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:42.118 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:42.115+0000 7f22f61bb640 1 -- 172.21.15.78:0/1086037420 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f22f0095c20 msgr2=0x7f22f0072a40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:22:42.118 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:42.115+0000 7f22f61bb640 1 --2- 172.21.15.78:0/1086037420 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f22f0095c20 0x7f22f0072a40 secure :-1 s=READY pgs=61 cs=0 l=1 rev1=1 crypto rx=0x7f22e0033040 tx=0x7f22e0003f90 comp rx=0 tx=0).stop 2024-09-21T09:22:42.118 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:42.115+0000 7f22f61bb640 1 -- 172.21.15.78:0/1086037420 shutdown_connections 2024-09-21T09:22:42.118 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:42.115+0000 7f22f61bb640 1 --2- 172.21.15.78:0/1086037420 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f22f012c160 0x7f22f00734c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:42.118 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:42.115+0000 7f22f61bb640 1 --2- 172.21.15.78:0/1086037420 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f22f00991f0 0x7f22f0072f80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:42.118 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:42.115+0000 7f22f61bb640 1 --2- 172.21.15.78:0/1086037420 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f22f0095c20 0x7f22f0072a40 unknown :-1 s=CLOSED pgs=61 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:22:42.118 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:42.115+0000 7f22f61bb640 1 -- 172.21.15.78:0/1086037420 >> 172.21.15.78:0/1086037420 conn(0x7f22f009a2e0 msgr2=0x7f22f008bf80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:22:42.118 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:42.115+0000 7f22f61bb640 1 -- 172.21.15.78:0/1086037420 shutdown_connections 2024-09-21T09:22:42.118 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:22:42.115+0000 7f22f61bb640 1 -- 172.21.15.78:0/1086037420 wait complete. 2024-09-21T09:22:42.129 INFO:teuthology.orchestra.run.smithi078.stdout:{"pg_ready":true,"pg_map":{"version":27,"stamp":"2024-09-21T09:22:40.402635+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":151,"num_write_kb":1896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":107,"ondisk_log_size":107,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":377487360,"kb_used":112236,"kb_used_data":2224,"kb_used_omap":17,"kb_used_meta":109934,"kb_avail":377375124,"statfs":{"total":386547056640,"available":386432126976,"internally_reserved":0,"allocated":2277376,"data_stored":1556658,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":17745,"internal_metadata":112573103},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.925598"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":49,"reported_epoch":32,"state":"active+clean","last_fresh":"2024-09-21T09:22:34.490846+0000","last_change":"2024-09-21T09:22:31.543815+0000","last_active":"2024-09-21T09:22:34.490846+0000","last_peered":"2024-09-21T09:22:34.490846+0000","last_clean":"2024-09-21T09:22:34.490846+0000","last_became_active":"2024-09-21T09:22:31.543451+0000","last_became_peered":"2024-09-21T09:22:31.543451+0000","last_unstale":"2024-09-21T09:22:34.490846+0000","last_undegraded":"2024-09-21T09:22:34.490846+0000","last_fullsized":"2024-09-21T09:22:34.490846+0000","mapping_epoch":28,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":29,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:25:48.514441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039605,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":40,"reported_epoch":32,"state":"active+clean","last_fresh":"2024-09-21T09:22:34.490419+0000","last_change":"2024-09-21T09:22:29.466574+0000","last_active":"2024-09-21T09:22:34.490419+0000","last_peered":"2024-09-21T09:22:34.490419+0000","last_clean":"2024-09-21T09:22:34.490419+0000","last_became_active":"2024-09-21T09:22:29.466289+0000","last_became_peered":"2024-09-21T09:22:29.466289+0000","last_unstale":"2024-09-21T09:22:34.490419+0000","last_undegraded":"2024-09-21T09:22:34.490419+0000","last_fullsized":"2024-09-21T09:22:34.490419+0000","mapping_epoch":26,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":27,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:11:52.267440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040851400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":49,"reported_epoch":32,"state":"active+clean","last_fresh":"2024-09-21T09:22:34.490724+0000","last_change":"2024-09-21T09:22:31.543852+0000","last_active":"2024-09-21T09:22:34.490724+0000","last_peered":"2024-09-21T09:22:34.490724+0000","last_clean":"2024-09-21T09:22:34.490724+0000","last_became_active":"2024-09-21T09:22:31.543538+0000","last_became_peered":"2024-09-21T09:22:31.543538+0000","last_unstale":"2024-09-21T09:22:34.490724+0000","last_undegraded":"2024-09-21T09:22:34.490724+0000","last_fullsized":"2024-09-21T09:22:34.490724+0000","mapping_epoch":28,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":29,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:04:07.910766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001005517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":40,"reported_epoch":32,"state":"active+clean","last_fresh":"2024-09-21T09:22:34.490274+0000","last_change":"2024-09-21T09:22:29.467296+0000","last_active":"2024-09-21T09:22:34.490274+0000","last_peered":"2024-09-21T09:22:34.490274+0000","last_clean":"2024-09-21T09:22:34.490274+0000","last_became_active":"2024-09-21T09:22:29.466923+0000","last_became_peered":"2024-09-21T09:22:29.466923+0000","last_unstale":"2024-09-21T09:22:34.490274+0000","last_undegraded":"2024-09-21T09:22:34.490274+0000","last_fullsized":"2024-09-21T09:22:34.490274+0000","mapping_epoch":26,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":27,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:59:07.385986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054560899999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"17'2","reported_seq":46,"reported_epoch":32,"state":"active+clean","last_fresh":"2024-09-21T09:22:34.490675+0000","last_change":"2024-09-21T09:22:31.472650+0000","last_active":"2024-09-21T09:22:34.490675+0000","last_peered":"2024-09-21T09:22:34.490675+0000","last_clean":"2024-09-21T09:22:34.490675+0000","last_became_active":"2024-09-21T09:22:31.472357+0000","last_became_peered":"2024-09-21T09:22:31.472357+0000","last_unstale":"2024-09-21T09:22:34.490675+0000","last_undegraded":"2024-09-21T09:22:34.490675+0000","last_fullsized":"2024-09-21T09:22:34.490675+0000","mapping_epoch":28,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":29,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:07:36.189205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011881540000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":37,"reported_epoch":32,"state":"active+clean","last_fresh":"2024-09-21T09:22:34.489357+0000","last_change":"2024-09-21T09:22:31.472256+0000","last_active":"2024-09-21T09:22:34.489357+0000","last_peered":"2024-09-21T09:22:34.489357+0000","last_clean":"2024-09-21T09:22:34.489357+0000","last_became_active":"2024-09-21T09:22:31.471983+0000","last_became_peered":"2024-09-21T09:22:31.471983+0000","last_unstale":"2024-09-21T09:22:34.489357+0000","last_undegraded":"2024-09-21T09:22:34.489357+0000","last_fullsized":"2024-09-21T09:22:34.489357+0000","mapping_epoch":28,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":29,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:01:34.840425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023051899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":44,"reported_epoch":32,"state":"active+clean","last_fresh":"2024-09-21T09:22:34.490328+0000","last_change":"2024-09-21T09:22:31.473209+0000","last_active":"2024-09-21T09:22:34.490328+0000","last_peered":"2024-09-21T09:22:34.490328+0000","last_clean":"2024-09-21T09:22:34.490328+0000","last_became_active":"2024-09-21T09:22:31.472948+0000","last_became_peered":"2024-09-21T09:22:31.472948+0000","last_unstale":"2024-09-21T09:22:34.490328+0000","last_undegraded":"2024-09-21T09:22:34.490328+0000","last_fullsized":"2024-09-21T09:22:34.490328+0000","mapping_epoch":28,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":29,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:44:51.405569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039429200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"15'1","reported_seq":42,"reported_epoch":32,"state":"active+clean","last_fresh":"2024-09-21T09:22:34.490515+0000","last_change":"2024-09-21T09:22:31.472253+0000","last_active":"2024-09-21T09:22:34.490515+0000","last_peered":"2024-09-21T09:22:34.490515+0000","last_clean":"2024-09-21T09:22:34.490515+0000","last_became_active":"2024-09-21T09:22:31.471933+0000","last_became_peered":"2024-09-21T09:22:31.471933+0000","last_unstale":"2024-09-21T09:22:34.490515+0000","last_undegraded":"2024-09-21T09:22:34.490515+0000","last_fullsized":"2024-09-21T09:22:34.490515+0000","mapping_epoch":28,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":29,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:55.301824+0000","last_clean_scrub_stamp":"2024-09-21T09:20:55.301824+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:57:08.422651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091827900000000004,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"20'104","reported_seq":190,"reported_epoch":32,"state":"active+clean","last_fresh":"2024-09-21T09:22:34.490159+0000","last_change":"2024-09-21T09:22:29.466956+0000","last_active":"2024-09-21T09:22:34.490159+0000","last_peered":"2024-09-21T09:22:34.490159+0000","last_clean":"2024-09-21T09:22:34.490159+0000","last_became_active":"2024-09-21T09:22:29.466649+0000","last_became_peered":"2024-09-21T09:22:29.466649+0000","last_unstale":"2024-09-21T09:22:34.490159+0000","last_undegraded":"2024-09-21T09:22:34.490159+0000","last_fullsized":"2024-09-21T09:22:34.490159+0000","mapping_epoch":26,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":27,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:20:51.278939+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:20:51.278939+0000","last_clean_scrub_stamp":"2024-09-21T09:20:51.278939+0000","objects_scrubbed":0,"log_size":104,"log_dups_size":0,"ondisk_log_size":104,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:02:40.936727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":149,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":4},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":149,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104,"ondisk_log_size":104,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":3,"up_from":28,"seq":120259084292,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27808,"kb_used_data":268,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":94344032,"statfs":{"total":96636764160,"available":96608288768,"internally_reserved":0,"allocated":274432,"data_stored":93990,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4435,"internal_metadata":28176045},"hb_peers":[0,1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":27,"seq":115964116996,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27796,"kb_used_data":264,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":94344044,"statfs":{"total":96636764160,"available":96608301056,"internally_reserved":0,"allocated":270336,"data_stored":93971,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4437,"internal_metadata":28176043},"hb_peers":[0,1,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":26,"seq":111669149700,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":28320,"kb_used_data":848,"kb_used_omap":4,"kb_used_meta":27451,"kb_avail":94343520,"statfs":{"total":96636764160,"available":96607764480,"internally_reserved":0,"allocated":868352,"data_stored":684358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4436,"internal_metadata":28110508},"hb_peers":[0,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":26,"seq":111669149700,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":28312,"kb_used_data":844,"kb_used_omap":4,"kb_used_meta":27451,"kb_avail":94343528,"statfs":{"total":96636764160,"available":96607772672,"internally_reserved":0,"allocated":864256,"data_stored":684339,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4437,"internal_metadata":28110507},"hb_peers":[1,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T09:22:42.131 INFO:tasks.ceph:pgid 2.7 last_scrub_stamp 2024-09-21T09:20:55.301824+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=21, tm_hour=9, tm_min=20, tm_sec=55, tm_wday=5, tm_yday=265, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=21, tm_hour=9, tm_min=22, tm_sec=39, tm_wday=5, tm_yday=265, tm_isdst=0) 2024-09-21T09:22:42.131 INFO:tasks.ceph:pgid 2.6 last_scrub_stamp 2024-09-21T09:20:55.301824+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=21, tm_hour=9, tm_min=20, tm_sec=55, tm_wday=5, tm_yday=265, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=21, tm_hour=9, tm_min=22, tm_sec=39, tm_wday=5, tm_yday=265, tm_isdst=0) 2024-09-21T09:22:42.132 INFO:tasks.ceph:pgid 2.5 last_scrub_stamp 2024-09-21T09:20:55.301824+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=21, tm_hour=9, tm_min=20, tm_sec=55, tm_wday=5, tm_yday=265, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=21, tm_hour=9, tm_min=22, tm_sec=39, tm_wday=5, tm_yday=265, tm_isdst=0) 2024-09-21T09:22:42.132 INFO:tasks.ceph:pgid 2.4 last_scrub_stamp 2024-09-21T09:20:55.301824+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=21, tm_hour=9, tm_min=20, tm_sec=55, tm_wday=5, tm_yday=265, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=21, tm_hour=9, tm_min=22, tm_sec=39, tm_wday=5, tm_yday=265, tm_isdst=0) 2024-09-21T09:22:42.132 INFO:tasks.ceph:pgid 2.2 last_scrub_stamp 2024-09-21T09:20:55.301824+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=21, tm_hour=9, tm_min=20, tm_sec=55, tm_wday=5, tm_yday=265, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=21, tm_hour=9, tm_min=22, tm_sec=39, tm_wday=5, tm_yday=265, tm_isdst=0) 2024-09-21T09:22:42.132 INFO:tasks.ceph:pgid 2.1 last_scrub_stamp 2024-09-21T09:20:55.301824+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=21, tm_hour=9, tm_min=20, tm_sec=55, tm_wday=5, tm_yday=265, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=21, tm_hour=9, tm_min=22, tm_sec=39, tm_wday=5, tm_yday=265, tm_isdst=0) 2024-09-21T09:22:42.132 INFO:tasks.ceph:pgid 2.0 last_scrub_stamp 2024-09-21T09:20:55.301824+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=21, tm_hour=9, tm_min=20, tm_sec=55, tm_wday=5, tm_yday=265, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=21, tm_hour=9, tm_min=22, tm_sec=39, tm_wday=5, tm_yday=265, tm_isdst=0) 2024-09-21T09:22:42.132 INFO:tasks.ceph:pgid 2.3 last_scrub_stamp 2024-09-21T09:20:55.301824+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=21, tm_hour=9, tm_min=20, tm_sec=55, tm_wday=5, tm_yday=265, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=21, tm_hour=9, tm_min=22, tm_sec=39, tm_wday=5, tm_yday=265, tm_isdst=0) 2024-09-21T09:22:42.132 INFO:tasks.ceph:pgid 1.0 last_scrub_stamp 2024-09-21T09:20:51.278939+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=21, tm_hour=9, tm_min=20, tm_sec=51, tm_wday=5, tm_yday=265, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=21, tm_hour=9, tm_min=22, tm_sec=39, tm_wday=5, tm_yday=265, tm_isdst=0) 2024-09-21T09:22:42.132 INFO:tasks.ceph:Still waiting for all pgs to be scrubbed. 2024-09-21T09:23:02.133 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-09-21T09:23:02.244 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.242+0000 7fb315f7d640 1 -- 172.21.15.78:0/2773177721 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fb310095110 msgr2=0x7fb310134c10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:23:02.244 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.242+0000 7fb315f7d640 1 --2- 172.21.15.78:0/2773177721 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fb310095110 0x7fb310134c10 secure :-1 s=READY pgs=62 cs=0 l=1 rev1=1 crypto rx=0x7fb308009830 tx=0x7fb30802a3f0 comp rx=0 tx=0).stop 2024-09-21T09:23:02.244 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.242+0000 7fb315f7d640 1 -- 172.21.15.78:0/2773177721 shutdown_connections 2024-09-21T09:23:02.244 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.242+0000 7fb315f7d640 1 --2- 172.21.15.78:0/2773177721 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fb310095110 0x7fb310134c10 unknown :-1 s=CLOSED pgs=62 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:23:02.244 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.242+0000 7fb315f7d640 1 --2- 172.21.15.78:0/2773177721 >> v2:172.21.15.78:3300/0 conn(0x7fb31012cc30 0x7fb31012d010 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:23:02.244 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.242+0000 7fb315f7d640 1 -- 172.21.15.78:0/2773177721 >> 172.21.15.78:0/2773177721 conn(0x7fb31009a2e0 msgr2=0x7fb310097ad0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:23:02.245 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.242+0000 7fb315f7d640 1 -- 172.21.15.78:0/2773177721 shutdown_connections 2024-09-21T09:23:02.245 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.242+0000 7fb315f7d640 1 -- 172.21.15.78:0/2773177721 wait complete. 2024-09-21T09:23:02.246 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.243+0000 7fb315f7d640 1 Processor -- start 2024-09-21T09:23:02.246 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.243+0000 7fb315f7d640 1 -- start start 2024-09-21T09:23:02.246 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.244+0000 7fb315f7d640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fb3100945d0 0x7fb31008c230 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:23:02.247 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.244+0000 7fb315f7d640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fb310095110 0x7fb31008c770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:23:02.247 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.244+0000 7fb315f7d640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fb31012cc30 0x7fb31008ccb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:23:02.247 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.244+0000 7fb315f7d640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7fb310135850 con 0x7fb310095110 2024-09-21T09:23:02.247 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.244+0000 7fb315f7d640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7fb3101356d0 con 0x7fb31012cc30 2024-09-21T09:23:02.247 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.244+0000 7fb315f7d640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7fb3101359d0 con 0x7fb3100945d0 2024-09-21T09:23:02.247 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.244+0000 7fb30f7fe640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fb3100945d0 0x7fb31008c230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:23:02.247 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.244+0000 7fb30f7fe640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fb3100945d0 0x7fb31008c230 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3302/0 says I am v2:172.21.15.78:34798/0 (socket says 172.21.15.78:34798) 2024-09-21T09:23:02.247 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.244+0000 7fb30f7fe640 1 -- 172.21.15.78:0/598368007 learned_addr learned my addr 172.21.15.78:0/598368007 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:23:02.247 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.244+0000 7fb30effd640 1 --2- 172.21.15.78:0/598368007 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fb310095110 0x7fb31008c770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:23:02.247 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.244+0000 7fb30ffff640 1 --2- 172.21.15.78:0/598368007 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fb31012cc30 0x7fb31008ccb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:23:02.247 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.244+0000 7fb30f7fe640 1 -- 172.21.15.78:0/598368007 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fb31012cc30 msgr2=0x7fb31008ccb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:23:02.247 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.244+0000 7fb30f7fe640 1 --2- 172.21.15.78:0/598368007 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fb31012cc30 0x7fb31008ccb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:23:02.247 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.244+0000 7fb30f7fe640 1 -- 172.21.15.78:0/598368007 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fb310095110 msgr2=0x7fb31008c770 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:23:02.248 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.244+0000 7fb30f7fe640 1 --2- 172.21.15.78:0/598368007 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fb310095110 0x7fb31008c770 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:23:02.248 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.244+0000 7fb30f7fe640 1 -- 172.21.15.78:0/598368007 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fb31008f2b0 con 0x7fb3100945d0 2024-09-21T09:23:02.248 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.244+0000 7fb30f7fe640 1 --2- 172.21.15.78:0/598368007 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fb3100945d0 0x7fb31008c230 secure :-1 s=READY pgs=63 cs=0 l=1 rev1=1 crypto rx=0x7fb30000c420 tx=0x7fb300009f90 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:23:02.248 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.244+0000 7fb2f7fff640 1 -- 172.21.15.78:0/598368007 <== mon.2 v2:172.21.15.78:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb300003970 con 0x7fb3100945d0 2024-09-21T09:23:02.248 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.245+0000 7fb2f7fff640 1 -- 172.21.15.78:0/598368007 <== mon.2 v2:172.21.15.78:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fb300003c00 con 0x7fb3100945d0 2024-09-21T09:23:02.248 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.245+0000 7fb315f7d640 1 -- 172.21.15.78:0/598368007 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fb310134ea0 con 0x7fb3100945d0 2024-09-21T09:23:02.248 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.245+0000 7fb2f7fff640 1 -- 172.21.15.78:0/598368007 <== mon.2 v2:172.21.15.78:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb300004020 con 0x7fb3100945d0 2024-09-21T09:23:02.248 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.245+0000 7fb315f7d640 1 -- 172.21.15.78:0/598368007 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7fb31008edd0 con 0x7fb3100945d0 2024-09-21T09:23:02.249 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.246+0000 7fb315f7d640 1 -- 172.21.15.78:0/598368007 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fb31012d010 con 0x7fb3100945d0 2024-09-21T09:23:02.249 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.246+0000 7fb2f7fff640 1 -- 172.21.15.78:0/598368007 <== mon.2 v2:172.21.15.78:3302/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7fb30001d020 con 0x7fb3100945d0 2024-09-21T09:23:02.249 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.246+0000 7fb2f7fff640 1 -- 172.21.15.78:0/598368007 <== mon.2 v2:172.21.15.78:3302/0 5 ==== osd_map(32..32 src has 1..32) ==== 3027+0+0 (secure 0 0 0) 0x7fb300069630 con 0x7fb3100945d0 2024-09-21T09:23:02.251 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.250+0000 7fb2f7fff640 1 -- 172.21.15.78:0/598368007 <== mon.2 v2:172.21.15.78:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7fb3000686e0 con 0x7fb3100945d0 2024-09-21T09:23:02.397 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.395+0000 7fb315f7d640 1 -- 172.21.15.78:0/598368007 --> v1:172.21.15.78:6816/2186157269 -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) -- 0x7fb3100699c0 con 0x7fb2fc0514e0 2024-09-21T09:23:02.397 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.396+0000 7fb2f7fff640 1 -- 172.21.15.78:0/598368007 <== mgr.14103 v1:172.21.15.78:6816/2186157269 1 ==== mgr_command_reply(tid 0: 0 dumped all) ==== 18+0+32001 (unknown 2965378022 0 473295874) 0x7fb3100699c0 con 0x7fb2fc0514e0 2024-09-21T09:23:02.398 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T09:23:02.398 INFO:teuthology.orchestra.run.smithi078.stderr:dumped all 2024-09-21T09:23:02.399 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.398+0000 7fb315f7d640 1 -- 172.21.15.78:0/598368007 >> v1:172.21.15.78:6816/2186157269 conn(0x7fb2fc0514e0 legacy=0x7fb2fc0539a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:23:02.399 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.398+0000 7fb315f7d640 1 -- 172.21.15.78:0/598368007 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fb3100945d0 msgr2=0x7fb31008c230 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:23:02.399 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.398+0000 7fb315f7d640 1 --2- 172.21.15.78:0/598368007 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fb3100945d0 0x7fb31008c230 secure :-1 s=READY pgs=63 cs=0 l=1 rev1=1 crypto rx=0x7fb30000c420 tx=0x7fb300009f90 comp rx=0 tx=0).stop 2024-09-21T09:23:02.400 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.398+0000 7fb315f7d640 1 -- 172.21.15.78:0/598368007 shutdown_connections 2024-09-21T09:23:02.400 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.398+0000 7fb315f7d640 1 --2- 172.21.15.78:0/598368007 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7fb31012cc30 0x7fb31008ccb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:23:02.400 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.398+0000 7fb315f7d640 1 --2- 172.21.15.78:0/598368007 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7fb310095110 0x7fb31008c770 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:23:02.400 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.398+0000 7fb315f7d640 1 --2- 172.21.15.78:0/598368007 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7fb3100945d0 0x7fb31008c230 unknown :-1 s=CLOSED pgs=63 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:23:02.400 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.398+0000 7fb315f7d640 1 -- 172.21.15.78:0/598368007 >> 172.21.15.78:0/598368007 conn(0x7fb31009a2e0 msgr2=0x7fb310129400 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:23:02.400 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.398+0000 7fb315f7d640 1 -- 172.21.15.78:0/598368007 shutdown_connections 2024-09-21T09:23:02.400 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.398+0000 7fb315f7d640 1 -- 172.21.15.78:0/598368007 wait complete. 2024-09-21T09:23:02.410 INFO:teuthology.orchestra.run.smithi078.stdout:{"pg_ready":true,"pg_map":{"version":37,"stamp":"2024-09-21T09:23:00.406141+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":151,"num_write_kb":1896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":110,"ondisk_log_size":110,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":377487360,"kb_used":112208,"kb_used_data":2224,"kb_used_omap":17,"kb_used_meta":109934,"kb_avail":377375152,"statfs":{"total":386547056640,"available":386432155648,"internally_reserved":0,"allocated":2277376,"data_stored":1556658,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":17745,"internal_metadata":112573103},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002291"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":57,"reported_epoch":32,"state":"active+clean","last_fresh":"2024-09-21T09:22:44.234990+0000","last_change":"2024-09-21T09:22:44.234990+0000","last_active":"2024-09-21T09:22:44.234990+0000","last_peered":"2024-09-21T09:22:44.234990+0000","last_clean":"2024-09-21T09:22:44.234990+0000","last_became_active":"2024-09-21T09:22:31.543451+0000","last_became_peered":"2024-09-21T09:22:31.543451+0000","last_unstale":"2024-09-21T09:22:44.234990+0000","last_undegraded":"2024-09-21T09:22:44.234990+0000","last_fullsized":"2024-09-21T09:22:44.234990+0000","mapping_epoch":28,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":29,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:22:44.234893+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:22:44.234893+0000","last_clean_scrub_stamp":"2024-09-21T09:22:44.234893+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:31:36.455235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039605,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":48,"reported_epoch":32,"state":"active+clean","last_fresh":"2024-09-21T09:22:43.250901+0000","last_change":"2024-09-21T09:22:43.250901+0000","last_active":"2024-09-21T09:22:43.250901+0000","last_peered":"2024-09-21T09:22:43.250901+0000","last_clean":"2024-09-21T09:22:43.250901+0000","last_became_active":"2024-09-21T09:22:29.466289+0000","last_became_peered":"2024-09-21T09:22:29.466289+0000","last_unstale":"2024-09-21T09:22:43.250901+0000","last_undegraded":"2024-09-21T09:22:43.250901+0000","last_fullsized":"2024-09-21T09:22:43.250901+0000","mapping_epoch":26,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":27,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:22:43.250808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:22:43.250808+0000","last_clean_scrub_stamp":"2024-09-21T09:22:43.250808+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:05:44.618744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040851400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":57,"reported_epoch":32,"state":"active+clean","last_fresh":"2024-09-21T09:22:45.271806+0000","last_change":"2024-09-21T09:22:45.271806+0000","last_active":"2024-09-21T09:22:45.271806+0000","last_peered":"2024-09-21T09:22:45.271806+0000","last_clean":"2024-09-21T09:22:45.271806+0000","last_became_active":"2024-09-21T09:22:31.543538+0000","last_became_peered":"2024-09-21T09:22:31.543538+0000","last_unstale":"2024-09-21T09:22:45.271806+0000","last_undegraded":"2024-09-21T09:22:45.271806+0000","last_fullsized":"2024-09-21T09:22:45.271806+0000","mapping_epoch":28,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":29,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:22:45.271690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:22:45.271690+0000","last_clean_scrub_stamp":"2024-09-21T09:22:45.271690+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:29:01.002783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001005517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":48,"reported_epoch":32,"state":"active+clean","last_fresh":"2024-09-21T09:22:42.233703+0000","last_change":"2024-09-21T09:22:42.233703+0000","last_active":"2024-09-21T09:22:42.233703+0000","last_peered":"2024-09-21T09:22:42.233703+0000","last_clean":"2024-09-21T09:22:42.233703+0000","last_became_active":"2024-09-21T09:22:29.466923+0000","last_became_peered":"2024-09-21T09:22:29.466923+0000","last_unstale":"2024-09-21T09:22:42.233703+0000","last_undegraded":"2024-09-21T09:22:42.233703+0000","last_fullsized":"2024-09-21T09:22:42.233703+0000","mapping_epoch":26,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":27,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:22:42.233609+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:22:42.233609+0000","last_clean_scrub_stamp":"2024-09-21T09:22:42.233609+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:07:06.278277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054560899999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"32'3","reported_seq":55,"reported_epoch":32,"state":"active+clean","last_fresh":"2024-09-21T09:22:42.312735+0000","last_change":"2024-09-21T09:22:42.312735+0000","last_active":"2024-09-21T09:22:42.312735+0000","last_peered":"2024-09-21T09:22:42.312735+0000","last_clean":"2024-09-21T09:22:42.312735+0000","last_became_active":"2024-09-21T09:22:31.472357+0000","last_became_peered":"2024-09-21T09:22:31.472357+0000","last_unstale":"2024-09-21T09:22:42.312735+0000","last_undegraded":"2024-09-21T09:22:42.312735+0000","last_fullsized":"2024-09-21T09:22:42.312735+0000","mapping_epoch":28,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":29,"parent":"0.0","parent_split_bits":0,"last_scrub":"32'3","last_scrub_stamp":"2024-09-21T09:22:42.312669+0000","last_deep_scrub":"32'3","last_deep_scrub_stamp":"2024-09-21T09:22:42.312669+0000","last_clean_scrub_stamp":"2024-09-21T09:22:42.312669+0000","objects_scrubbed":1,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:55:48.985222+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0.0011881540000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":45,"reported_epoch":32,"state":"active+clean","last_fresh":"2024-09-21T09:22:41.919442+0000","last_change":"2024-09-21T09:22:41.919442+0000","last_active":"2024-09-21T09:22:41.919442+0000","last_peered":"2024-09-21T09:22:41.919442+0000","last_clean":"2024-09-21T09:22:41.919442+0000","last_became_active":"2024-09-21T09:22:31.471983+0000","last_became_peered":"2024-09-21T09:22:31.471983+0000","last_unstale":"2024-09-21T09:22:41.919442+0000","last_undegraded":"2024-09-21T09:22:41.919442+0000","last_fullsized":"2024-09-21T09:22:41.919442+0000","mapping_epoch":28,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":29,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:22:41.919365+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:22:41.919365+0000","last_clean_scrub_stamp":"2024-09-21T09:22:41.919365+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:32:00.939250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023051899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":52,"reported_epoch":32,"state":"active+clean","last_fresh":"2024-09-21T09:22:46.273416+0000","last_change":"2024-09-21T09:22:46.273416+0000","last_active":"2024-09-21T09:22:46.273416+0000","last_peered":"2024-09-21T09:22:46.273416+0000","last_clean":"2024-09-21T09:22:46.273416+0000","last_became_active":"2024-09-21T09:22:31.472948+0000","last_became_peered":"2024-09-21T09:22:31.472948+0000","last_unstale":"2024-09-21T09:22:46.273416+0000","last_undegraded":"2024-09-21T09:22:46.273416+0000","last_fullsized":"2024-09-21T09:22:46.273416+0000","mapping_epoch":28,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":29,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:22:46.273308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:22:46.273308+0000","last_clean_scrub_stamp":"2024-09-21T09:22:46.273308+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:53:53.256598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039429200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"32'2","reported_seq":51,"reported_epoch":32,"state":"active+clean","last_fresh":"2024-09-21T09:22:43.280167+0000","last_change":"2024-09-21T09:22:43.280167+0000","last_active":"2024-09-21T09:22:43.280167+0000","last_peered":"2024-09-21T09:22:43.280167+0000","last_clean":"2024-09-21T09:22:43.280167+0000","last_became_active":"2024-09-21T09:22:31.471933+0000","last_became_peered":"2024-09-21T09:22:31.471933+0000","last_unstale":"2024-09-21T09:22:43.280167+0000","last_undegraded":"2024-09-21T09:22:43.280167+0000","last_fullsized":"2024-09-21T09:22:43.280167+0000","mapping_epoch":28,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":29,"parent":"0.0","parent_split_bits":0,"last_scrub":"32'2","last_scrub_stamp":"2024-09-21T09:22:43.280075+0000","last_deep_scrub":"32'2","last_deep_scrub_stamp":"2024-09-21T09:22:43.280075+0000","last_clean_scrub_stamp":"2024-09-21T09:22:43.280075+0000","objects_scrubbed":1,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:04:41.032954+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0.00091827900000000004,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"32'105","reported_seq":199,"reported_epoch":32,"state":"active+clean","last_fresh":"2024-09-21T09:22:41.189905+0000","last_change":"2024-09-21T09:22:41.189905+0000","last_active":"2024-09-21T09:22:41.189905+0000","last_peered":"2024-09-21T09:22:41.189905+0000","last_clean":"2024-09-21T09:22:41.189905+0000","last_became_active":"2024-09-21T09:22:29.466649+0000","last_became_peered":"2024-09-21T09:22:29.466649+0000","last_unstale":"2024-09-21T09:22:41.189905+0000","last_undegraded":"2024-09-21T09:22:41.189905+0000","last_fullsized":"2024-09-21T09:22:41.189905+0000","mapping_epoch":26,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":27,"parent":"0.0","parent_split_bits":0,"last_scrub":"32'105","last_scrub_stamp":"2024-09-21T09:22:41.189828+0000","last_deep_scrub":"32'105","last_deep_scrub_stamp":"2024-09-21T09:22:41.189828+0000","last_clean_scrub_stamp":"2024-09-21T09:22:41.189828+0000","objects_scrubbed":2,"log_size":105,"log_dups_size":0,"ondisk_log_size":105,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:02:31.046090+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":149,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":5,"ondisk_log_size":5,"up":16,"acting":16,"num_store_stats":4},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":149,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":105,"ondisk_log_size":105,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":3,"up_from":28,"seq":120259084296,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27796,"kb_used_data":268,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":94344044,"statfs":{"total":96636764160,"available":96608301056,"internally_reserved":0,"allocated":274432,"data_stored":93990,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4435,"internal_metadata":28176045},"hb_peers":[0,1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":27,"seq":115964117000,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27792,"kb_used_data":264,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":94344048,"statfs":{"total":96636764160,"available":96608305152,"internally_reserved":0,"allocated":270336,"data_stored":93971,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4437,"internal_metadata":28176043},"hb_peers":[0,1,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":26,"seq":111669149704,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":28312,"kb_used_data":848,"kb_used_omap":4,"kb_used_meta":27451,"kb_avail":94343528,"statfs":{"total":96636764160,"available":96607772672,"internally_reserved":0,"allocated":868352,"data_stored":684358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4436,"internal_metadata":28110508},"hb_peers":[0,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":26,"seq":111669149704,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":28308,"kb_used_data":844,"kb_used_omap":4,"kb_used_meta":27451,"kb_avail":94343532,"statfs":{"total":96636764160,"available":96607776768,"internally_reserved":0,"allocated":864256,"data_stored":684339,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4437,"internal_metadata":28110507},"hb_peers":[1,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T09:23:02.411 DEBUG:teuthology.orchestra.run.smithi078:> sudo ceph --cluster ceph config set global mon_health_to_clog false 2024-09-21T09:23:02.512 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.510+0000 7f01ec108640 1 -- 172.21.15.78:0/3101109994 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f01e40710e0 msgr2=0x7f01e4130e30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:23:02.512 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.510+0000 7f01ec108640 1 --2- 172.21.15.78:0/3101109994 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f01e40710e0 0x7f01e4130e30 secure :-1 s=READY pgs=148 cs=0 l=1 rev1=1 crypto rx=0x7f01d4009900 tx=0x7f01d402a4f0 comp rx=0 tx=0).stop 2024-09-21T09:23:02.512 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.510+0000 7f01ec108640 1 -- 172.21.15.78:0/3101109994 shutdown_connections 2024-09-21T09:23:02.512 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.510+0000 7f01ec108640 1 --2- 172.21.15.78:0/3101109994 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f01e4131370 0x7f01e4133760 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:23:02.512 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.510+0000 7f01ec108640 1 --2- 172.21.15.78:0/3101109994 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f01e40710e0 0x7f01e4130e30 unknown :-1 s=CLOSED pgs=148 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:23:02.512 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.510+0000 7f01ec108640 1 -- 172.21.15.78:0/3101109994 >> 172.21.15.78:0/3101109994 conn(0x7f01e409dec0 msgr2=0x7f01e409e2d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:23:02.512 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.511+0000 7f01ec108640 1 -- 172.21.15.78:0/3101109994 shutdown_connections 2024-09-21T09:23:02.512 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.511+0000 7f01ec108640 1 -- 172.21.15.78:0/3101109994 wait complete. 2024-09-21T09:23:02.514 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.511+0000 7f01ec108640 1 Processor -- start 2024-09-21T09:23:02.514 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.511+0000 7f01ec108640 1 -- start start 2024-09-21T09:23:02.514 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.512+0000 7f01ec108640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f01e40705f0 0x7f01e4126610 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:23:02.514 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.512+0000 7f01ec108640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f01e40710e0 0x7f01e4126b50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:23:02.514 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.512+0000 7f01ec108640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f01e4131370 0x7f01e4127090 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:23:02.514 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.512+0000 7f01ec108640 1 -- --> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] -- mon_getmap magic: 0 -- 0x7f01e41347b0 con 0x7f01e40710e0 2024-09-21T09:23:02.515 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.512+0000 7f01ec108640 1 -- --> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] -- mon_getmap magic: 0 -- 0x7f01e4134630 con 0x7f01e4131370 2024-09-21T09:23:02.515 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.512+0000 7f01ec108640 1 -- --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_getmap magic: 0 -- 0x7f01e4134930 con 0x7f01e40705f0 2024-09-21T09:23:02.515 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.512+0000 7f01e96a3640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f01e40710e0 0x7f01e4126b50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:23:02.515 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.512+0000 7f01e9ea4640 1 --2- >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f01e40705f0 0x7f01e4126610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:23:02.515 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.512+0000 7f01ea6a5640 1 --2- >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f01e4131370 0x7f01e4127090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:23:02.515 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.512+0000 7f01e96a3640 1 --2- >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f01e40710e0 0x7f01e4126b50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.78:3300/0 says I am v2:172.21.15.78:49670/0 (socket says 172.21.15.78:49670) 2024-09-21T09:23:02.515 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.512+0000 7f01e96a3640 1 -- 172.21.15.78:0/4105541190 learned_addr learned my addr 172.21.15.78:0/4105541190 (peer_addr_for_me v2:172.21.15.78:0/0) 2024-09-21T09:23:02.515 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.512+0000 7f01e9ea4640 1 -- 172.21.15.78:0/4105541190 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f01e4131370 msgr2=0x7f01e4127090 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:23:02.515 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.512+0000 7f01e9ea4640 1 --2- 172.21.15.78:0/4105541190 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f01e4131370 0x7f01e4127090 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:23:02.515 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.512+0000 7f01e9ea4640 1 -- 172.21.15.78:0/4105541190 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f01e40710e0 msgr2=0x7f01e4126b50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:23:02.515 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.512+0000 7f01e9ea4640 1 --2- 172.21.15.78:0/4105541190 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f01e40710e0 0x7f01e4126b50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:23:02.515 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.512+0000 7f01e9ea4640 1 -- 172.21.15.78:0/4105541190 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f01e406f750 con 0x7f01e40705f0 2024-09-21T09:23:02.515 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.512+0000 7f01e96a3640 1 --2- 172.21.15.78:0/4105541190 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f01e40710e0 0x7f01e4126b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:23:02.516 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.512+0000 7f01e9ea4640 1 --2- 172.21.15.78:0/4105541190 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f01e40705f0 0x7f01e4126610 secure :-1 s=READY pgs=64 cs=0 l=1 rev1=1 crypto rx=0x7f01e000c430 tx=0x7f01e000c8f0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:23:02.516 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.513+0000 7f01da7fc640 1 -- 172.21.15.78:0/4105541190 <== mon.2 v2:172.21.15.78:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f01e0017070 con 0x7f01e40705f0 2024-09-21T09:23:02.516 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.513+0000 7f01da7fc640 1 -- 172.21.15.78:0/4105541190 <== mon.2 v2:172.21.15.78:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f01e00053c0 con 0x7f01e40705f0 2024-09-21T09:23:02.516 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.513+0000 7f01ec108640 1 -- 172.21.15.78:0/4105541190 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f01e406fcd0 con 0x7f01e40705f0 2024-09-21T09:23:02.516 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.513+0000 7f01da7fc640 1 -- 172.21.15.78:0/4105541190 <== mon.2 v2:172.21.15.78:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f01e0003850 con 0x7f01e40705f0 2024-09-21T09:23:02.516 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.513+0000 7f01ec108640 1 -- 172.21.15.78:0/4105541190 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f01e4128090 con 0x7f01e40705f0 2024-09-21T09:23:02.516 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.514+0000 7f01ec108640 1 -- 172.21.15.78:0/4105541190 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f01e40709d0 con 0x7f01e40705f0 2024-09-21T09:23:02.517 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.514+0000 7f01da7fc640 1 -- 172.21.15.78:0/4105541190 <== mon.2 v2:172.21.15.78:3302/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f01e001a070 con 0x7f01e40705f0 2024-09-21T09:23:02.517 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.514+0000 7f01da7fc640 1 -- 172.21.15.78:0/4105541190 <== mon.2 v2:172.21.15.78:3302/0 5 ==== osd_map(32..32 src has 1..32) ==== 3027+0+0 (secure 0 0 0) 0x7f01e00683b0 con 0x7f01e40705f0 2024-09-21T09:23:02.519 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.518+0000 7f01da7fc640 1 -- 172.21.15.78:0/4105541190 <== mon.2 v2:172.21.15.78:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f01e0067460 con 0x7f01e40705f0 2024-09-21T09:23:02.668 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.667+0000 7f01ec108640 1 -- 172.21.15.78:0/4105541190 --> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] -- mon_command([{prefix=config set, name=mon_health_to_clog}] v 0) -- 0x7f01e40698c0 con 0x7f01e40705f0 2024-09-21T09:23:02.673 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.671+0000 7f01da7fc640 1 -- 172.21.15.78:0/4105541190 <== mon.2 v2:172.21.15.78:3302/0 7 ==== config(6 keys) ==== 172+0+0 (secure 0 0 0) 0x7f01e0044810 con 0x7f01e40705f0 2024-09-21T09:23:02.673 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.671+0000 7f01da7fc640 1 -- 172.21.15.78:0/4105541190 <== mon.2 v2:172.21.15.78:3302/0 8 ==== mon_command_ack([{prefix=config set, name=mon_health_to_clog}]=0 v24)=0 v24) ==== 123+0+0 (secure 0 0 0) 0x7f01e0066fa0 con 0x7f01e40705f0 2024-09-21T09:23:02.675 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.673+0000 7f01ec108640 1 -- 172.21.15.78:0/4105541190 >> v1:172.21.15.78:6816/2186157269 conn(0x7f01c80514b0 legacy=0x7f01c8053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:23:02.675 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.673+0000 7f01ec108640 1 -- 172.21.15.78:0/4105541190 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f01e40705f0 msgr2=0x7f01e4126610 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:23:02.675 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.673+0000 7f01ec108640 1 --2- 172.21.15.78:0/4105541190 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f01e40705f0 0x7f01e4126610 secure :-1 s=READY pgs=64 cs=0 l=1 rev1=1 crypto rx=0x7f01e000c430 tx=0x7f01e000c8f0 comp rx=0 tx=0).stop 2024-09-21T09:23:02.675 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.674+0000 7f01ec108640 1 -- 172.21.15.78:0/4105541190 shutdown_connections 2024-09-21T09:23:02.675 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.674+0000 7f01ec108640 1 --2- 172.21.15.78:0/4105541190 >> [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] conn(0x7f01e4131370 0x7f01e4127090 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:23:02.675 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.674+0000 7f01ec108640 1 --2- 172.21.15.78:0/4105541190 >> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] conn(0x7f01e40710e0 0x7f01e4126b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:23:02.675 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.674+0000 7f01ec108640 1 --2- 172.21.15.78:0/4105541190 >> [v2:172.21.15.78:3302/0,v1:172.21.15.78:6791/0] conn(0x7f01e40705f0 0x7f01e4126610 unknown :-1 s=CLOSED pgs=64 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:23:02.676 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.674+0000 7f01ec108640 1 -- 172.21.15.78:0/4105541190 >> 172.21.15.78:0/4105541190 conn(0x7f01e409dec0 msgr2=0x7f01e4132ba0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:23:02.676 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.674+0000 7f01ec108640 1 -- 172.21.15.78:0/4105541190 shutdown_connections 2024-09-21T09:23:02.676 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-21T09:23:02.674+0000 7f01ec108640 1 -- 172.21.15.78:0/4105541190 wait complete. 2024-09-21T09:23:02.686 INFO:teuthology.misc:Shutting down mds daemons... 2024-09-21T09:23:02.686 INFO:teuthology.misc:Shutting down osd daemons... 2024-09-21T09:23:02.687 DEBUG:tasks.ceph.osd.0:waiting for process to exit 2024-09-21T09:23:02.687 INFO:teuthology.orchestra.run:waiting for 300 2024-09-21T09:23:02.719 INFO:tasks.ceph.osd.0:Stopped 2024-09-21T09:23:02.719 DEBUG:tasks.ceph.osd.1:waiting for process to exit 2024-09-21T09:23:02.719 INFO:teuthology.orchestra.run:waiting for 300 2024-09-21T09:23:02.791 INFO:tasks.ceph.osd.1:Stopped 2024-09-21T09:23:02.791 DEBUG:tasks.ceph.osd.2:waiting for process to exit 2024-09-21T09:23:02.791 INFO:teuthology.orchestra.run:waiting for 300 2024-09-21T09:23:02.858 INFO:tasks.ceph.osd.2:Stopped 2024-09-21T09:23:02.859 DEBUG:tasks.ceph.osd.3:waiting for process to exit 2024-09-21T09:23:02.859 INFO:teuthology.orchestra.run:waiting for 300 2024-09-21T09:23:02.925 INFO:tasks.ceph.osd.3:Stopped 2024-09-21T09:23:02.925 INFO:teuthology.misc:Shutting down mgr daemons... 2024-09-21T09:23:02.926 DEBUG:tasks.ceph.mgr.x:waiting for process to exit 2024-09-21T09:23:02.926 INFO:teuthology.orchestra.run:waiting for 300 2024-09-21T09:23:02.991 INFO:tasks.ceph.mgr.x:Stopped 2024-09-21T09:23:02.991 DEBUG:tasks.ceph.mgr.y:waiting for process to exit 2024-09-21T09:23:02.991 INFO:teuthology.orchestra.run:waiting for 300 2024-09-21T09:23:03.054 INFO:tasks.ceph.mgr.y:Stopped 2024-09-21T09:23:03.054 INFO:teuthology.misc:Shutting down mon daemons... 2024-09-21T09:23:03.054 DEBUG:tasks.ceph.mon.a:waiting for process to exit 2024-09-21T09:23:03.055 INFO:teuthology.orchestra.run:waiting for 300 2024-09-21T09:23:03.055 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.mgr.x is failed for ~0s 2024-09-21T09:23:03.055 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.mgr.y is failed for ~0s 2024-09-21T09:23:03.106 INFO:tasks.ceph.mon.a:Stopped 2024-09-21T09:23:03.106 DEBUG:tasks.ceph.mon.b:waiting for process to exit 2024-09-21T09:23:03.106 INFO:teuthology.orchestra.run:waiting for 300 2024-09-21T09:23:03.158 INFO:tasks.ceph.mon.b:Stopped 2024-09-21T09:23:03.158 DEBUG:tasks.ceph.mon.c:waiting for process to exit 2024-09-21T09:23:03.158 INFO:teuthology.orchestra.run:waiting for 300 2024-09-21T09:23:03.209 INFO:tasks.ceph.mon.c:Stopped 2024-09-21T09:23:03.209 INFO:tasks.ceph:Checking cluster log for badness... 2024-09-21T09:23:03.209 DEBUG:teuthology.orchestra.run.smithi078:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/ceph.log | egrep -v 'overall HEALTH_' | egrep -v '\(FS_' | egrep -v '\(MDS_' | egrep -v '\(OSD_' | egrep -v '\(MON_DOWN\)' | egrep -v '\(CACHE_POOL_' | egrep -v '\(POOL_' | egrep -v '\(MGR_DOWN\)' | egrep -v '\(PG_' | egrep -v '\(SMALLER_PGP_NUM\)' | egrep -v 'Monitor daemon marked osd' | egrep -v 'Behind on trimming' | egrep -v 'Manager daemon' | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v FS_DEGRADED | egrep -v 'fs.*is degraded' | egrep -v 'filesystem is degraded' | egrep -v FS_INLINE_DATA_DEPRECATED | egrep -v FS_WITH_FAILED_MDS | egrep -v MDS_ALL_DOWN | egrep -v 'filesystem is offline' | egrep -v MDS_DAMAGE | egrep -v MDS_DEGRADED | egrep -v MDS_FAILED | egrep -v MDS_INSUFFICIENT_STANDBY | egrep -v 'insufficient standby MDS daemons available' | egrep -v MDS_UP_LESS_THAN_MAX | egrep -v 'online, but wants' | egrep -v 'filesystem is online with fewer MDS than max_mds' | egrep -v POOL_APP_NOT_ENABLED | egrep -v 'do not have an application enabled' | egrep -v 'overall HEALTH_' | egrep -v 'Replacing daemon' | egrep -v 'deprecated feature inline_data' | egrep -v 'overall HEALTH_' | egrep -v '\(OSD_DOWN\)' | egrep -v '\(OSD_' | egrep -v 'but it is still running' | egrep -v 'is not responding' | egrep -v PG_AVAILABILITY | egrep -v PG_DEGRADED | egrep -v 'Reduced data availability' | egrep -v 'Degraded data redundancy' | egrep -v 'scrub mismatch' | egrep -v ScrubResult | egrep -v 'wrongly marked' | egrep -v '\(POOL_APP_NOT_ENABLED\)' | egrep -v '\(SLOW_OPS\)' | egrep -v 'overall HEALTH_' | egrep -v '\(MON_MSGR2_NOT_ENABLED\)' | egrep -v 'slow request' | head -n 1 2024-09-21T09:23:03.290 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-0 on ubuntu@smithi078.front.sepia.ceph.com 2024-09-21T09:23:03.290 DEBUG:teuthology.orchestra.run.smithi078:> sync && sudo umount -f /var/lib/ceph/osd/ceph-0 2024-09-21T09:23:03.487 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-1 on ubuntu@smithi078.front.sepia.ceph.com 2024-09-21T09:23:03.487 DEBUG:teuthology.orchestra.run.smithi078:> sync && sudo umount -f /var/lib/ceph/osd/ceph-1 2024-09-21T09:23:03.583 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-2 on ubuntu@smithi078.front.sepia.ceph.com 2024-09-21T09:23:03.584 DEBUG:teuthology.orchestra.run.smithi078:> sync && sudo umount -f /var/lib/ceph/osd/ceph-2 2024-09-21T09:23:03.642 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-3 on ubuntu@smithi078.front.sepia.ceph.com 2024-09-21T09:23:03.642 DEBUG:teuthology.orchestra.run.smithi078:> sync && sudo umount -f /var/lib/ceph/osd/ceph-3 2024-09-21T09:23:03.702 INFO:tasks.ceph:Archiving mon data... 2024-09-21T09:23:03.703 DEBUG:teuthology.misc:Transferring archived files from smithi078:/var/lib/ceph/mon/ceph-a to /home/teuthworker/archive/teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi/7913639/data/mon.a.tgz 2024-09-21T09:23:03.703 DEBUG:teuthology.orchestra.run.smithi078:> mktemp 2024-09-21T09:23:03.718 INFO:teuthology.orchestra.run.smithi078.stdout:/tmp/tmp.wBW6vYGnj0 2024-09-21T09:23:03.718 DEBUG:teuthology.orchestra.run.smithi078:> sudo tar cz -f - -C /var/lib/ceph/mon/ceph-a -- . > /tmp/tmp.wBW6vYGnj0 2024-09-21T09:23:03.839 DEBUG:teuthology.orchestra.run.smithi078:> sudo chmod 0666 /tmp/tmp.wBW6vYGnj0 2024-09-21T09:23:03.922 DEBUG:teuthology.orchestra.remote:smithi078:/tmp/tmp.wBW6vYGnj0 is 236KB 2024-09-21T09:23:04.014 DEBUG:teuthology.orchestra.run.smithi078:> rm -fr /tmp/tmp.wBW6vYGnj0 2024-09-21T09:23:04.029 DEBUG:teuthology.misc:Transferring archived files from smithi078:/var/lib/ceph/mon/ceph-b to /home/teuthworker/archive/teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi/7913639/data/mon.b.tgz 2024-09-21T09:23:04.029 DEBUG:teuthology.orchestra.run.smithi078:> mktemp 2024-09-21T09:23:04.082 INFO:teuthology.orchestra.run.smithi078.stdout:/tmp/tmp.48zpPjNQf1 2024-09-21T09:23:04.083 DEBUG:teuthology.orchestra.run.smithi078:> sudo tar cz -f - -C /var/lib/ceph/mon/ceph-b -- . > /tmp/tmp.48zpPjNQf1 2024-09-21T09:23:04.198 DEBUG:teuthology.orchestra.run.smithi078:> sudo chmod 0666 /tmp/tmp.48zpPjNQf1 2024-09-21T09:23:04.280 DEBUG:teuthology.orchestra.remote:smithi078:/tmp/tmp.48zpPjNQf1 is 282KB 2024-09-21T09:23:04.373 DEBUG:teuthology.orchestra.run.smithi078:> rm -fr /tmp/tmp.48zpPjNQf1 2024-09-21T09:23:04.388 DEBUG:teuthology.misc:Transferring archived files from smithi078:/var/lib/ceph/mon/ceph-c to /home/teuthworker/archive/teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi/7913639/data/mon.c.tgz 2024-09-21T09:23:04.388 DEBUG:teuthology.orchestra.run.smithi078:> mktemp 2024-09-21T09:23:04.444 INFO:teuthology.orchestra.run.smithi078.stdout:/tmp/tmp.UYL5DeYs6P 2024-09-21T09:23:04.444 DEBUG:teuthology.orchestra.run.smithi078:> sudo tar cz -f - -C /var/lib/ceph/mon/ceph-c -- . > /tmp/tmp.UYL5DeYs6P 2024-09-21T09:23:04.560 DEBUG:teuthology.orchestra.run.smithi078:> sudo chmod 0666 /tmp/tmp.UYL5DeYs6P 2024-09-21T09:23:04.643 DEBUG:teuthology.orchestra.remote:smithi078:/tmp/tmp.UYL5DeYs6P is 281KB 2024-09-21T09:23:04.736 DEBUG:teuthology.orchestra.run.smithi078:> rm -fr /tmp/tmp.UYL5DeYs6P 2024-09-21T09:23:04.751 INFO:tasks.ceph:Cleaning ceph cluster... 2024-09-21T09:23:04.751 DEBUG:teuthology.orchestra.run.smithi078:> sudo rm -rf -- /etc/ceph/ceph.conf /etc/ceph/ceph.keyring /home/ubuntu/cephtest/ceph.data /home/ubuntu/cephtest/ceph.monmap /home/ubuntu/cephtest/../*.pid 2024-09-21T09:23:04.869 INFO:teuthology.util.scanner:summary_data or yaml_file is empty! 2024-09-21T09:23:04.869 INFO:tasks.ceph:Archiving crash dumps... 2024-09-21T09:23:04.870 DEBUG:teuthology.misc:Transferring archived files from smithi078:/var/lib/ceph/crash to /home/teuthworker/archive/teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi/7913639/remote/smithi078/crash 2024-09-21T09:23:04.871 DEBUG:teuthology.orchestra.run.smithi078:> sudo tar c -f - -C /var/lib/ceph/crash -- . 2024-09-21T09:23:04.933 INFO:tasks.ceph:Compressing logs... 2024-09-21T09:23:04.933 DEBUG:teuthology.orchestra.run.smithi078:> time sudo find /var/log/ceph -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-21T09:23:05.009 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-osd.0.log 2024-09-21T09:23:05.009 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.32671.log 2024-09-21T09:23:05.009 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-osd.0.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39371.log 2024-09-21T09:23:05.009 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.32671.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47329.log 2024-09-21T09:23:05.010 INFO:teuthology.orchestra.run.smithi078.stderr: 76.1% -- replaced with /var/log/ceph/ceph-client.admin.32671.log.gz 2024-09-21T09:23:05.010 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.39371.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.41867.log 2024-09-21T09:23:05.010 INFO:teuthology.orchestra.run.smithi078.stderr: 86.6% -- replaced with /var/log/ceph/ceph-client.admin.39371.log.gz 2024-09-21T09:23:05.010 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46719.log 2024-09-21T09:23:05.010 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.47329.log: /var/log/ceph/ceph-client.admin.41867.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47497.log 2024-09-21T09:23:05.010 INFO:teuthology.orchestra.run.smithi078.stderr: 85.6% -- replaced with /var/log/ceph/ceph-client.admin.47329.log.gz 2024-09-21T09:23:05.010 INFO:teuthology.orchestra.run.smithi078.stderr: 87.1%/var/log/ceph/ceph-client.admin.46719.log: -- replaced with /var/log/ceph/ceph-client.admin.41867.log.gz 2024-09-21T09:23:05.010 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46718.log 2024-09-21T09:23:05.010 INFO:teuthology.orchestra.run.smithi078.stderr: 86.9% -- replaced with /var/log/ceph/ceph-client.admin.46719.log.gz 2024-09-21T09:23:05.010 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.47497.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.40209.log 2024-09-21T09:23:05.010 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.46718.log: 85.7% -- replaced with /var/log/ceph/ceph-client.admin.47497.log.gz 2024-09-21T09:23:05.010 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39471.log 2024-09-21T09:23:05.010 INFO:teuthology.orchestra.run.smithi078.stderr: 86.7% -- replaced with /var/log/ceph/ceph-client.admin.46718.log.gz 2024-09-21T09:23:05.010 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.40209.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47077.log 2024-09-21T09:23:05.011 INFO:teuthology.orchestra.run.smithi078.stderr: 86.4% -- replaced with /var/log/ceph/ceph-client.admin.40209.log.gz 2024-09-21T09:23:05.011 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.39471.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.32885.log 2024-09-21T09:23:05.011 INFO:teuthology.orchestra.run.smithi078.stderr: 85.9% -- replaced with /var/log/ceph/ceph-client.admin.39471.log.gz 2024-09-21T09:23:05.011 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.47077.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39809.log 2024-09-21T09:23:05.011 INFO:teuthology.orchestra.run.smithi078.stderr: 86.3% -- replaced with /var/log/ceph/ceph-client.admin.47077.log.gz 2024-09-21T09:23:05.011 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.32885.log: gzip -5 --verbose -- /var/log/ceph/ceph-osd.1.log 2024-09-21T09:23:05.011 INFO:teuthology.orchestra.run.smithi078.stderr: 86.7% -- replaced with /var/log/ceph/ceph-client.admin.32885.log.gz 2024-09-21T09:23:05.011 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.39809.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.32710.log 2024-09-21T09:23:05.011 INFO:teuthology.orchestra.run.smithi078.stderr: 86.7% -- replaced with /var/log/ceph/ceph-client.admin.39809.log.gz 2024-09-21T09:23:05.011 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.41847.log 2024-09-21T09:23:05.011 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.32710.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46758.log 2024-09-21T09:23:05.011 INFO:teuthology.orchestra.run.smithi078.stderr: 86.0% -- replaced with /var/log/ceph/ceph-client.admin.32710.log.gz 2024-09-21T09:23:05.011 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.41847.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39470.log 2024-09-21T09:23:05.011 INFO:teuthology.orchestra.run.smithi078.stderr: 86.7% -- replaced with /var/log/ceph/ceph-client.admin.41847.log.gz 2024-09-21T09:23:05.012 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.46758.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.42075.log 2024-09-21T09:23:05.012 INFO:teuthology.orchestra.run.smithi078.stderr: 86.3% -- replaced with /var/log/ceph/ceph-client.admin.46758.log.gz 2024-09-21T09:23:05.012 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46759.log 2024-09-21T09:23:05.012 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.39470.log: /var/log/ceph/ceph-client.admin.42075.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.41656.log 2024-09-21T09:23:05.012 INFO:teuthology.orchestra.run.smithi078.stderr: 86.1% -- replaced with /var/log/ceph/ceph-client.admin.39470.log.gz 2024-09-21T09:23:05.012 INFO:teuthology.orchestra.run.smithi078.stderr: 86.8% -- replaced with /var/log/ceph/ceph-client.admin.42075.log.gz 2024-09-21T09:23:05.012 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.46759.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.42097.log 2024-09-21T09:23:05.012 INFO:teuthology.orchestra.run.smithi078.stderr: 86.6% -- replaced with /var/log/ceph/ceph-client.admin.46759.log.gz 2024-09-21T09:23:05.012 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47623.log 2024-09-21T09:23:05.012 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.41656.log: /var/log/ceph/ceph-client.admin.42097.log: 87.1%gzip -- replaced with /var/log/ceph/ceph-client.admin.41656.log.gz 2024-09-21T09:23:05.012 INFO:teuthology.orchestra.run.smithi078.stderr: -5 --verbose -- /var/log/ceph/ceph-client.admin.40249.log 2024-09-21T09:23:05.012 INFO:teuthology.orchestra.run.smithi078.stderr: 86.5% -- replaced with /var/log/ceph/ceph-client.admin.42097.log.gz 2024-09-21T09:23:05.012 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.47623.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39005.log 2024-09-21T09:23:05.012 INFO:teuthology.orchestra.run.smithi078.stderr: 86.2% -- replaced with /var/log/ceph/ceph-client.admin.47623.log.gz 2024-09-21T09:23:05.013 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.42194.log 2024-09-21T09:23:05.013 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.40249.log: /var/log/ceph/ceph-client.admin.39005.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39990.log 2024-09-21T09:23:05.013 INFO:teuthology.orchestra.run.smithi078.stderr: 86.5% -- replaced with /var/log/ceph/ceph-client.admin.40249.log.gz 2024-09-21T09:23:05.013 INFO:teuthology.orchestra.run.smithi078.stderr: 86.9% -- replaced with /var/log/ceph/ceph-client.admin.39005.log.gz 2024-09-21T09:23:05.013 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.42194.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39769.log 2024-09-21T09:23:05.013 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.32536.log 2024-09-21T09:23:05.013 INFO:teuthology.orchestra.run.smithi078.stderr: 86.7% -- replaced with /var/log/ceph/ceph-client.admin.42194.log.gz 2024-09-21T09:23:05.013 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.39990.log: /var/log/ceph/ceph-client.admin.39769.log: 86.7% -- replaced with /var/log/ceph/ceph-client.admin.39990.log.gz 2024-09-21T09:23:05.013 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46317.log 2024-09-21T09:23:05.013 INFO:teuthology.orchestra.run.smithi078.stderr: 87.1% -- replaced with /var/log/ceph/ceph-client.admin.39769.log.gz 2024-09-21T09:23:05.013 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.32266.log 2024-09-21T09:23:05.013 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.32536.log: 75.6% -- replaced with /var/log/ceph/ceph-client.admin.32536.log.gz 2024-09-21T09:23:05.013 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.46317.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46928.log 2024-09-21T09:23:05.013 INFO:teuthology.orchestra.run.smithi078.stderr: 86.9% -- replaced with /var/log/ceph/ceph-client.admin.46317.log.gz 2024-09-21T09:23:05.013 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.32266.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47035.log 2024-09-21T09:23:05.014 INFO:teuthology.orchestra.run.smithi078.stderr: 0.0% -- replaced with /var/log/ceph/ceph-client.admin.32266.log.gz 2024-09-21T09:23:05.014 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.46928.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47413.log 2024-09-21T09:23:05.014 INFO:teuthology.orchestra.run.smithi078.stderr: 86.7% -- replaced with /var/log/ceph/ceph-client.admin.46928.log.gz 2024-09-21T09:23:05.014 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.47035.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.40169.log 2024-09-21T09:23:05.014 INFO:teuthology.orchestra.run.smithi078.stderr: 86.5% -- replaced with /var/log/ceph/ceph-client.admin.47035.log.gz 2024-09-21T09:23:05.014 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.47413.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47287.log 2024-09-21T09:23:05.014 INFO:teuthology.orchestra.run.smithi078.stderr: 85.5% -- replaced with /var/log/ceph/ceph-client.admin.47413.log.gz 2024-09-21T09:23:05.014 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.33480.log 2024-09-21T09:23:05.014 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.40169.log: gzip -5 --verbose/var/log/ceph/ceph-client.admin.47287.log: -- /var/log/ceph/ceph-client.admin.47455.log 2024-09-21T09:23:05.014 INFO:teuthology.orchestra.run.smithi078.stderr: 87.3% -- replaced with /var/log/ceph/ceph-client.admin.40169.log.gz 2024-09-21T09:23:05.014 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.33480.log: 87.0% -- replaced with /var/log/ceph/ceph-client.admin.47287.log.gz 2024-09-21T09:23:05.014 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.41947.log 2024-09-21T09:23:05.014 INFO:teuthology.orchestra.run.smithi078.stderr: 87.0% -- replaced with /var/log/ceph/ceph-client.admin.33480.log.gz 2024-09-21T09:23:05.014 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.47455.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39113.log 2024-09-21T09:23:05.015 INFO:teuthology.orchestra.run.smithi078.stderr: 85.7% -- replaced with /var/log/ceph/ceph-client.admin.47455.log.gz 2024-09-21T09:23:05.015 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.41947.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39287.log 2024-09-21T09:23:05.015 INFO:teuthology.orchestra.run.smithi078.stderr: 87.1% -- replaced with /var/log/ceph/ceph-client.admin.41947.log.gz 2024-09-21T09:23:05.015 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.39113.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.41614.log 2024-09-21T09:23:05.015 INFO:teuthology.orchestra.run.smithi078.stderr: 86.8% -- replaced with /var/log/ceph/ceph-client.admin.39113.log.gz 2024-09-21T09:23:05.015 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.39287.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.42404.log 2024-09-21T09:23:05.015 INFO:teuthology.orchestra.run.smithi078.stderr: 86.5% -- replaced with /var/log/ceph/ceph-client.admin.39287.log.gz 2024-09-21T09:23:05.035 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.41614.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39639.log 2024-09-21T09:23:05.035 INFO:teuthology.orchestra.run.smithi078.stderr: 87.2% -- replaced with /var/log/ceph/ceph-client.admin.41614.log.gz 2024-09-21T09:23:05.035 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.42404.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39640.log 2024-09-21T09:23:05.035 INFO:teuthology.orchestra.run.smithi078.stderr: 86.0% -- replaced with /var/log/ceph/ceph-client.admin.42404.log.gz 2024-09-21T09:23:05.035 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.39639.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46591.log 2024-09-21T09:23:05.035 INFO:teuthology.orchestra.run.smithi078.stderr: 86.7% -- replaced with /var/log/ceph/ceph-client.admin.39639.log.gz 2024-09-21T09:23:05.035 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.32508.log 2024-09-21T09:23:05.035 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.39640.log: /var/log/ceph/ceph-client.admin.46591.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.42550.log 2024-09-21T09:23:05.035 INFO:teuthology.orchestra.run.smithi078.stderr: 87.1% -- replaced with /var/log/ceph/ceph-client.admin.39640.log.gz 2024-09-21T09:23:05.035 INFO:teuthology.orchestra.run.smithi078.stderr: 85.6% -- replaced with /var/log/ceph/ceph-client.admin.46591.log.gz 2024-09-21T09:23:05.036 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.32508.log: 0.0% -- replaced with /var/log/ceph/ceph-client.admin.32508.log.gz 2024-09-21T09:23:05.036 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46489.log 2024-09-21T09:23:05.036 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/ceph.audit.log 2024-09-21T09:23:05.036 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.42550.log: /var/log/ceph/ceph-client.admin.46489.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46590.log 2024-09-21T09:23:05.036 INFO:teuthology.orchestra.run.smithi078.stderr: 86.8% -- replaced with /var/log/ceph/ceph-client.admin.42550.log.gz 2024-09-21T09:23:05.036 INFO:teuthology.orchestra.run.smithi078.stderr: 87.0% -- replaced with /var/log/ceph/ceph-client.admin.46489.log.gz 2024-09-21T09:23:05.036 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46405.log 2024-09-21T09:23:05.036 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47245.log 2024-09-21T09:23:05.036 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.46590.log: /var/log/ceph/ceph-client.admin.46405.log: 85.9% -- replaced with /var/log/ceph/ceph-client.admin.46590.log.gz 2024-09-21T09:23:05.036 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- 86.3% /var/log/ceph/ceph-client.admin.41479.log 2024-09-21T09:23:05.036 INFO:teuthology.orchestra.run.smithi078.stderr: -- replaced with /var/log/ceph/ceph-client.admin.46405.log.gz 2024-09-21T09:23:05.036 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.47245.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.41477.log 2024-09-21T09:23:05.036 INFO:teuthology.orchestra.run.smithi078.stderr: 86.3% -- replaced with /var/log/ceph/ceph-client.admin.47245.log.gz 2024-09-21T09:23:05.036 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.41479.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46363.log 2024-09-21T09:23:05.037 INFO:teuthology.orchestra.run.smithi078.stderr: 95.0% -- replaced with /var/log/ceph/ceph.audit.log.gz 2024-09-21T09:23:05.037 INFO:teuthology.orchestra.run.smithi078.stderr: 85.1% -- replaced with /var/log/ceph/ceph-client.admin.41479.log.gz 2024-09-21T09:23:05.037 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.41477.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39641.log 2024-09-21T09:23:05.037 INFO:teuthology.orchestra.run.smithi078.stderr: 85.3% -- replaced with /var/log/ceph/ceph-client.admin.41477.log.gz 2024-09-21T09:23:05.037 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.46363.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39638.log 2024-09-21T09:23:05.037 INFO:teuthology.orchestra.run.smithi078.stderr: 86.5% -- replaced with /var/log/ceph/ceph-client.admin.46363.log.gz 2024-09-21T09:23:05.037 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47747.log 2024-09-21T09:23:05.037 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.39641.log: /var/log/ceph/ceph-client.admin.39638.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47707.log 2024-09-21T09:23:05.037 INFO:teuthology.orchestra.run.smithi078.stderr: 86.6% -- replaced with /var/log/ceph/ceph-client.admin.39641.log.gz 2024-09-21T09:23:05.037 INFO:teuthology.orchestra.run.smithi078.stderr: 86.9% -- replaced with /var/log/ceph/ceph-client.admin.39638.log.gz 2024-09-21T09:23:05.037 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.47747.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.32888.log 2024-09-21T09:23:05.037 INFO:teuthology.orchestra.run.smithi078.stderr: 86.5% -- replaced with /var/log/ceph/ceph-client.admin.47747.log.gz 2024-09-21T09:23:05.037 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.47707.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46156.log 2024-09-21T09:23:05.037 INFO:teuthology.orchestra.run.smithi078.stderr: 85.9% -- replaced with /var/log/ceph/ceph-client.admin.47707.log.gz 2024-09-21T09:23:05.038 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.32888.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.32480.log 2024-09-21T09:23:05.038 INFO:teuthology.orchestra.run.smithi078.stderr: 85.9% -- replaced with /var/log/ceph/ceph-client.admin.32888.log.gz 2024-09-21T09:23:05.038 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.46156.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.42637.log 2024-09-21T09:23:05.038 INFO:teuthology.orchestra.run.smithi078.stderr: 86.7% -- replaced with /var/log/ceph/ceph-client.admin.46156.log.gz 2024-09-21T09:23:05.038 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.32480.log: 0.0% -- replaced with /var/log/ceph/ceph-client.admin.32480.log.gz 2024-09-21T09:23:05.038 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.41908.log 2024-09-21T09:23:05.038 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47161.log 2024-09-21T09:23:05.038 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.42637.log: /var/log/ceph/ceph-client.admin.41908.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.44880.log 2024-09-21T09:23:05.038 INFO:teuthology.orchestra.run.smithi078.stderr: 87.1% -- replaced with /var/log/ceph/ceph-client.admin.42637.log.gz 2024-09-21T09:23:05.038 INFO:teuthology.orchestra.run.smithi078.stderr: 86.8% -- replaced with /var/log/ceph/ceph-client.admin.41908.log.gz 2024-09-21T09:23:05.038 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.47161.log: gzip -5 --verbose -- /var/log/ceph/ceph-mon.c.log 2024-09-21T09:23:05.038 INFO:teuthology.orchestra.run.smithi078.stderr: 86.0% -- replaced with /var/log/ceph/ceph-client.admin.47161.log.gz 2024-09-21T09:23:05.038 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.44880.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39768.log 2024-09-21T09:23:05.038 INFO:teuthology.orchestra.run.smithi078.stderr: 86.4% -- replaced with /var/log/ceph/ceph-client.admin.44880.log.gz 2024-09-21T09:23:05.039 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.32442.log 2024-09-21T09:23:05.039 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-mon.c.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39194.log 2024-09-21T09:23:05.039 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.39768.log: /var/log/ceph/ceph-client.admin.32442.log: 86.5% -- replaced with /var/log/ceph/ceph-client.admin.39768.log.gz 2024-09-21T09:23:05.039 INFO:teuthology.orchestra.run.smithi078.stderr: 0.0% -- replaced with /var/log/ceph/ceph-client.admin.32442.log.gz 2024-09-21T09:23:05.039 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.32772.log 2024-09-21T09:23:05.039 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.42274.log 2024-09-21T09:23:05.039 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.39194.log: /var/log/ceph/ceph-client.admin.32772.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46236.log 2024-09-21T09:23:05.039 INFO:teuthology.orchestra.run.smithi078.stderr: 85.2% -- replaced with /var/log/ceph/ceph-client.admin.39194.log.gz 86.1% -- replaced with /var/log/ceph/ceph-client.admin.32772.log.gz 2024-09-21T09:23:05.039 INFO:teuthology.orchestra.run.smithi078.stderr: 2024-09-21T09:23:05.039 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.42274.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39473.log 2024-09-21T09:23:05.039 INFO:teuthology.orchestra.run.smithi078.stderr: 86.9% -- replaced with /var/log/ceph/ceph-client.admin.42274.log.gz 2024-09-21T09:23:05.039 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.42234.log 2024-09-21T09:23:05.039 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.46236.log: /var/log/ceph/ceph-client.admin.39473.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.32761.log 2024-09-21T09:23:05.039 INFO:teuthology.orchestra.run.smithi078.stderr: 86.7% -- replaced with /var/log/ceph/ceph-client.admin.46236.log.gz 2024-09-21T09:23:05.039 INFO:teuthology.orchestra.run.smithi078.stderr: 85.9% -- replaced with /var/log/ceph/ceph-client.admin.39473.log.gz 2024-09-21T09:23:05.040 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.42234.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.41252.log 2024-09-21T09:23:05.040 INFO:teuthology.orchestra.run.smithi078.stderr: 86.5% -- replaced with /var/log/ceph/ceph-client.admin.42234.log.gz 2024-09-21T09:23:05.040 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-mon.a.log 2024-09-21T09:23:05.040 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.32761.log: /var/log/ceph/ceph-client.admin.41252.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46867.log 2024-09-21T09:23:05.040 INFO:teuthology.orchestra.run.smithi078.stderr: 86.2% -- replaced with /var/log/ceph/ceph-client.admin.32761.log.gz 2024-09-21T09:23:05.040 INFO:teuthology.orchestra.run.smithi078.stderr: 87.0% -- replaced with /var/log/ceph/ceph-client.admin.41252.log.gz 2024-09-21T09:23:05.040 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-mon.a.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.40127.log 2024-09-21T09:23:05.040 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.33362.log 2024-09-21T09:23:05.040 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.46867.log: /var/log/ceph/ceph-client.admin.40127.log: gzip 87.0% -- replaced with /var/log/ceph/ceph-client.admin.46867.log.gz 2024-09-21T09:23:05.040 INFO:teuthology.orchestra.run.smithi078.stderr: -5 --verbose -- /var/log/ceph/ceph-client.admin.46887.log 2024-09-21T09:23:05.040 INFO:teuthology.orchestra.run.smithi078.stderr: 86.2% -- replaced with /var/log/ceph/ceph-client.admin.40127.log.gz 2024-09-21T09:23:05.040 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.33362.log: gzip -5 --verbose -- /var/log/ceph/ceph-osd.2.log 2024-09-21T09:23:05.040 INFO:teuthology.orchestra.run.smithi078.stderr: 86.7% -- replaced with /var/log/ceph/ceph-client.admin.33362.log.gz 2024-09-21T09:23:05.040 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.46887.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39472.log 2024-09-21T09:23:05.041 INFO:teuthology.orchestra.run.smithi078.stderr: 86.7% -- replaced with /var/log/ceph/ceph-client.admin.46887.log.gz 2024-09-21T09:23:05.041 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-osd.2.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.42015.log 2024-09-21T09:23:05.041 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5/var/log/ceph/ceph-client.admin.39472.log: --verbose -- /var/log/ceph/ceph-client.admin.46276.log 2024-09-21T09:23:05.041 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.42015.log: 85.7% -- replaced with /var/log/ceph/ceph-client.admin.39472.log.gz 2024-09-21T09:23:05.041 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39245.log 2024-09-21T09:23:05.041 INFO:teuthology.orchestra.run.smithi078.stderr: 86.7% -- replaced with /var/log/ceph/ceph-client.admin.42015.log.gz 2024-09-21T09:23:05.041 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.46276.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39956.log 2024-09-21T09:23:05.041 INFO:teuthology.orchestra.run.smithi078.stderr: 87.0% -- replaced with /var/log/ceph/ceph-client.admin.46276.log.gz/var/log/ceph/ceph-client.admin.39245.log: 2024-09-21T09:23:05.041 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- 86.7% -- replaced with /var/log/ceph/ceph-client.admin.39245.log.gz 2024-09-21T09:23:05.041 INFO:teuthology.orchestra.run.smithi078.stderr: /var/log/ceph/ceph-client.admin.42818.log 2024-09-21T09:23:05.041 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.39956.log: gzip -5 87.0% -- replaced with /var/log/ceph/ceph-client.admin.39956.log.gz 2024-09-21T09:23:05.041 INFO:teuthology.orchestra.run.smithi078.stderr: --verbose -- /var/log/ceph/ceph-client.admin.47203.log 2024-09-21T09:23:05.041 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.42818.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.42362.log 2024-09-21T09:23:05.041 INFO:teuthology.orchestra.run.smithi078.stderr: 86.5% -- replaced with /var/log/ceph/ceph-client.admin.42818.log.gz 2024-09-21T09:23:05.041 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.47203.log: gzip -5 --verbose -- /var/log/ceph/ceph-mgr.y.log 2024-09-21T09:23:05.042 INFO:teuthology.orchestra.run.smithi078.stderr: 86.3% -- replaced with /var/log/ceph/ceph-client.admin.47203.log.gz 2024-09-21T09:23:05.042 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.41335.log/var/log/ceph/ceph-client.admin.42362.log: 2024-09-21T09:23:05.042 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-mgr.y.log: 86.4% -- replaced with /var/log/ceph/ceph-client.admin.42362.log.gz 2024-09-21T09:23:05.042 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.42488.log 2024-09-21T09:23:05.042 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.32403.log 2024-09-21T09:23:05.042 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.42488.log: /var/log/ceph/ceph-client.admin.41335.log: 86.5% -- replaced with /var/log/ceph/ceph-client.admin.41335.log.gz 2024-09-21T09:23:05.042 INFO:teuthology.orchestra.run.smithi078.stderr: 86.4% -- replaced with /var/log/ceph/ceph-client.admin.42488.log.gz 2024-09-21T09:23:05.042 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/ceph.log 2024-09-21T09:23:05.042 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.32403.log: 74.6% -- replaced with /var/log/ceph/ceph-client.admin.32403.log.gz 2024-09-21T09:23:05.042 INFO:teuthology.orchestra.run.smithi078.stderr: 91.8% -- replaced with /var/log/ceph/ceph-mgr.y.log.gz 2024-09-21T09:23:05.043 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.40085.log 2024-09-21T09:23:05.044 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46447.log 2024-09-21T09:23:05.045 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.40085.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47119.log 2024-09-21T09:23:05.045 INFO:teuthology.orchestra.run.smithi078.stderr: 86.0% -- replaced with /var/log/ceph/ceph-client.admin.40085.log.gz 2024-09-21T09:23:05.045 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.46447.log: 93.8% -- replaced with /var/log/ceph/ceph.log.gz 2024-09-21T09:23:05.045 INFO:teuthology.orchestra.run.smithi078.stderr: 86.5% -- replaced with /var/log/ceph/ceph-client.admin.46447.log.gz 2024-09-21T09:23:05.045 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47539.log 2024-09-21T09:23:05.063 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.47119.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.32876.log 2024-09-21T09:23:05.064 INFO:teuthology.orchestra.run.smithi078.stderr: 86.5% -- replaced with /var/log/ceph/ceph-client.admin.47119.log.gz 2024-09-21T09:23:05.064 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.47539.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.42446.log 2024-09-21T09:23:05.064 INFO:teuthology.orchestra.run.smithi078.stderr: 86.4% -- replaced with /var/log/ceph/ceph-client.admin.47539.log.gz 2024-09-21T09:23:05.064 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.32876.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.33322.log 2024-09-21T09:23:05.064 INFO:teuthology.orchestra.run.smithi078.stderr: 85.9% -- replaced with /var/log/ceph/ceph-client.admin.32876.log.gz 2024-09-21T09:23:05.064 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.42446.log: gzip -5 --verbose 86.6% -- replaced with /var/log/ceph/ceph-client.admin.42446.log.gz 2024-09-21T09:23:05.064 INFO:teuthology.orchestra.run.smithi078.stderr: -- /var/log/ceph/ceph-client.admin.46947.log 2024-09-21T09:23:05.064 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.33322.log: gzip -5 --verbose 87.0% -- replaced with /var/log/ceph/ceph-client.admin.33322.log.gz 2024-09-21T09:23:05.064 INFO:teuthology.orchestra.run.smithi078.stderr: -- /var/log/ceph/ceph-client.admin.47665.log 2024-09-21T09:23:05.064 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.46947.log: gzip -5 --verbose -- /var/log/ceph/ceph-mgr.x.log 87.1% -- replaced with /var/log/ceph/ceph-client.admin.46947.log.gz 2024-09-21T09:23:05.064 INFO:teuthology.orchestra.run.smithi078.stderr: 2024-09-21T09:23:05.064 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.47665.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.32243.log 86.2% -- replaced with /var/log/ceph/ceph-client.admin.47665.log.gz 2024-09-21T09:23:05.064 INFO:teuthology.orchestra.run.smithi078.stderr: 2024-09-21T09:23:05.065 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.42037.log 2024-09-21T09:23:05.065 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.32243.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46116.log 2024-09-21T09:23:05.065 INFO:teuthology.orchestra.run.smithi078.stderr: 86.5% -- replaced with /var/log/ceph/ceph-client.admin.32243.log.gz 2024-09-21T09:23:05.065 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.42037.log: 93.6% -- replaced with /var/log/ceph/ceph-osd.0.log.gz 2024-09-21T09:23:05.065 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39169.log 2024-09-21T09:23:05.065 INFO:teuthology.orchestra.run.smithi078.stderr: 86.7% -- replaced with /var/log/ceph/ceph-client.admin.42037.log.gz 2024-09-21T09:23:05.065 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.46116.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.33282.log 2024-09-21T09:23:05.065 INFO:teuthology.orchestra.run.smithi078.stderr: 86.9% -- replaced with /var/log/ceph/ceph-client.admin.46116.log.gz 2024-09-21T09:23:05.065 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47581.log 2024-09-21T09:23:05.065 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.39169.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.33573.log 2024-09-21T09:23:05.065 INFO:teuthology.orchestra.run.smithi078.stderr: 86.8% -- replaced with /var/log/ceph/ceph-client.admin.39169.log.gz 2024-09-21T09:23:05.065 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.33282.log: /var/log/ceph/ceph-client.admin.47581.log: 87.0% -- replaced with /var/log/ceph/ceph-client.admin.33282.log.gzgzip 85.3% -- replaced with /var/log/ceph/ceph-client.admin.47581.log.gz 2024-09-21T09:23:05.065 INFO:teuthology.orchestra.run.smithi078.stderr: -5 --verbose -- /var/log/ceph/ceph-client.admin.41616.log 2024-09-21T09:23:05.065 INFO:teuthology.orchestra.run.smithi078.stderr: 2024-09-21T09:23:05.066 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.33573.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46196.log 87.0% -- replaced with /var/log/ceph/ceph-client.admin.33573.log.gz 2024-09-21T09:23:05.066 INFO:teuthology.orchestra.run.smithi078.stderr: 2024-09-21T09:23:05.066 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.41616.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.41471.log 2024-09-21T09:23:05.066 INFO:teuthology.orchestra.run.smithi078.stderr: 87.1% -- replaced with /var/log/ceph/ceph-client.admin.41616.log.gz 2024-09-21T09:23:05.066 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.46196.log: gzip -5 --verbose -- /var/log/ceph/ceph.tmp-client.admin.26402.log 2024-09-21T09:23:05.066 INFO:teuthology.orchestra.run.smithi078.stderr: 86.5% -- replaced with /var/log/ceph/ceph-client.admin.46196.log.gz 2024-09-21T09:23:05.066 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.41480.log 2024-09-21T09:23:05.066 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.41471.log: /var/log/ceph/ceph.tmp-client.admin.26402.log: gzip -5 --verbose -- /var/log/ceph/ceph-mon.b.log 2024-09-21T09:23:05.066 INFO:teuthology.orchestra.run.smithi078.stderr: 0.0% -- replaced with /var/log/ceph/ceph.tmp-client.admin.26402.log.gz 85.5% -- replaced with /var/log/ceph/ceph-client.admin.41471.log.gz 2024-09-21T09:23:05.066 INFO:teuthology.orchestra.run.smithi078.stderr: 2024-09-21T09:23:05.066 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.41480.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.42273.log 2024-09-21T09:23:05.066 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-mon.b.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.32917.log 2024-09-21T09:23:05.066 INFO:teuthology.orchestra.run.smithi078.stderr: 85.4% -- replaced with /var/log/ceph/ceph-client.admin.41480.log.gz 2024-09-21T09:23:05.066 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.42273.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.33242.log 2024-09-21T09:23:05.067 INFO:teuthology.orchestra.run.smithi078.stderr: 87.1% -- replaced with /var/log/ceph/ceph-client.admin.42273.log.gz 2024-09-21T09:23:05.067 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.32917.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.33130.log 2024-09-21T09:23:05.067 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.33242.log: 86.9% -- replaced with /var/log/ceph/ceph-client.admin.32917.log.gz 2024-09-21T09:23:05.067 INFO:teuthology.orchestra.run.smithi078.stderr: 86.6% -- replaced with /var/log/ceph/ceph-client.admin.33242.log.gzgzip -5 --verbose -- /var/log/ceph/ceph-client.admin.41655.log 2024-09-21T09:23:05.067 INFO:teuthology.orchestra.run.smithi078.stderr: 2024-09-21T09:23:05.067 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39917.log 2024-09-21T09:23:05.067 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.41655.log: /var/log/ceph/ceph-client.admin.33130.log: 86.9% -- replaced with /var/log/ceph/ceph-client.admin.33130.log.gz 2024-09-21T09:23:05.067 INFO:teuthology.orchestra.run.smithi078.stderr: 86.9% -- replaced with /var/log/ceph/ceph-client.admin.41655.log.gz 2024-09-21T09:23:05.067 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47371.log 2024-09-21T09:23:05.067 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.39917.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.32913.log 2024-09-21T09:23:05.067 INFO:teuthology.orchestra.run.smithi078.stderr: 87.0% -- replaced with /var/log/ceph/ceph-client.admin.39917.log.gz 2024-09-21T09:23:05.067 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.47371.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46588.log 2024-09-21T09:23:05.067 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.32913.log: 86.4% -- replaced with /var/log/ceph/ceph-client.admin.47371.log.gz 2024-09-21T09:23:05.067 INFO:teuthology.orchestra.run.smithi078.stderr: 86.5% -- replaced with /var/log/ceph/ceph-client.admin.32913.log.gz 2024-09-21T09:23:05.067 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39329.log 2024-09-21T09:23:05.068 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.41378.log 2024-09-21T09:23:05.068 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.46588.log: /var/log/ceph/ceph-client.admin.39329.log: 85.5% -- replaced with /var/log/ceph/ceph-client.admin.46588.log.gz 2024-09-21T09:23:05.068 INFO:teuthology.orchestra.run.smithi078.stderr: 86.4% -- replaced with /var/log/ceph/ceph-client.admin.39329.log.gz 2024-09-21T09:23:05.068 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.42728.log 2024-09-21T09:23:05.068 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.41378.log: 86.7% -- replaced with /var/log/ceph/ceph-client.admin.41378.log.gz 2024-09-21T09:23:05.068 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.32564.log 2024-09-21T09:23:05.068 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.32919.log 2024-09-21T09:23:05.068 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.42728.log: /var/log/ceph/ceph-client.admin.32564.log: 86.5% -- replaced with /var/log/ceph/ceph-client.admin.42728.log.gz 2024-09-21T09:23:05.068 INFO:teuthology.orchestra.run.smithi078.stderr: 75.7% -- replaced with /var/log/ceph/ceph-client.admin.32564.log.gz 2024-09-21T09:23:05.068 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46340.log 2024-09-21T09:23:05.068 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-osd.3.log 2024-09-21T09:23:05.068 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.32919.log: /var/log/ceph/ceph-client.admin.46340.log: 86.1% -- replaced with /var/log/ceph/ceph-client.admin.32919.log.gz 2024-09-21T09:23:05.068 INFO:teuthology.orchestra.run.smithi078.stderr: 87.1% -- replaced with /var/log/ceph/ceph-client.admin.46340.log.gz 2024-09-21T09:23:05.068 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46589.log 2024-09-21T09:23:05.074 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-osd.3.log: 93.1% -- replaced with /var/log/ceph/ceph-osd.2.log.gz 2024-09-21T09:23:05.074 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39957.log 2024-09-21T09:23:05.075 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.46589.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39808.log 2024-09-21T09:23:05.075 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.39957.log: 85.7% -- replaced with /var/log/ceph/ceph-client.admin.46589.log.gz 2024-09-21T09:23:05.076 INFO:teuthology.orchestra.run.smithi078.stderr: 86.6% -- replaced with /var/log/ceph/ceph-client.admin.39957.log.gz 2024-09-21T09:23:05.076 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph-client.admin.39808.log: 87.2% -- replaced with /var/log/ceph/ceph-client.admin.39808.log.gz 2024-09-21T09:23:05.089 INFO:teuthology.orchestra.run.smithi078.stderr: 91.7% -- replaced with /var/log/ceph/ceph-mon.c.log.gz 2024-09-21T09:23:05.091 INFO:teuthology.orchestra.run.smithi078.stderr: 90.6% -- replaced with /var/log/ceph/ceph-mgr.x.log.gz 2024-09-21T09:23:05.097 INFO:teuthology.orchestra.run.smithi078.stderr: 93.7% -- replaced with /var/log/ceph/ceph-osd.1.log.gz 2024-09-21T09:23:05.110 INFO:teuthology.orchestra.run.smithi078.stderr: 91.7% -- replaced with /var/log/ceph/ceph-mon.b.log.gz 2024-09-21T09:23:05.124 INFO:teuthology.orchestra.run.smithi078.stderr: 93.3% -- replaced with /var/log/ceph/ceph-osd.3.log.gz 2024-09-21T09:23:05.148 INFO:teuthology.orchestra.run.smithi078.stderr: 90.9% -- replaced with /var/log/ceph/ceph-mon.a.log.gz 2024-09-21T09:23:05.150 INFO:teuthology.orchestra.run.smithi078.stderr: 2024-09-21T09:23:05.150 INFO:teuthology.orchestra.run.smithi078.stderr:real 0m0.162s 2024-09-21T09:23:05.150 INFO:teuthology.orchestra.run.smithi078.stderr:user 0m0.600s 2024-09-21T09:23:05.150 INFO:teuthology.orchestra.run.smithi078.stderr:sys 0m0.108s 2024-09-21T09:23:05.150 INFO:tasks.ceph:Archiving logs... 2024-09-21T09:23:05.150 DEBUG:teuthology.misc:Transferring archived files from smithi078:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi/7913639/remote/smithi078/log 2024-09-21T09:23:05.151 DEBUG:teuthology.orchestra.run.smithi078:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-21T09:23:05.394 DEBUG:teuthology.run_tasks:Unwinding manager install 2024-09-21T09:23:05.411 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2024-09-21T09:23:05.412 DEBUG:teuthology.orchestra.run.smithi078:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-09-21T09:23:05.465 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-09-21T09:23:05.475 INFO:teuthology.task.clock:Checking final clock skew... 2024-09-21T09:23:05.475 DEBUG:teuthology.orchestra.run.smithi078:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-21T09:23:05.492 INFO:teuthology.orchestra.run.smithi078.stderr:bash: line 1: ntpq: command not found 2024-09-21T09:23:05.496 INFO:teuthology.orchestra.run.smithi078.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-21T09:23:05.497 INFO:teuthology.orchestra.run.smithi078.stdout:=============================================================================== 2024-09-21T09:23:05.497 INFO:teuthology.orchestra.run.smithi078.stdout:^* hv01.front.sepia.ceph.com 3 6 177 55 +31us[-5277ns] +/- 47ms 2024-09-21T09:23:05.497 INFO:teuthology.orchestra.run.smithi078.stdout:^+ hv02.front.sepia.ceph.com 3 6 177 55 +329us[ +293us] +/- 60ms 2024-09-21T09:23:05.497 INFO:teuthology.orchestra.run.smithi078.stdout:^+ hv03.front.sepia.ceph.com 3 6 177 53 -397us[ -397us] +/- 60ms 2024-09-21T09:23:05.497 INFO:teuthology.orchestra.run.smithi078.stdout:^? hv04.front.sepia.ceph.com 0 8 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T09:23:05.497 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-09-21T09:23:05.508 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-09-21T09:23:05.508 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-09-21T09:23:05.517 DEBUG:teuthology.orchestra.run.smithi078:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-21T09:23:05.600 DEBUG:teuthology.orchestra.run.smithi078:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-21T09:23:05.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:23:05.627 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-09-21T09:23:05.644 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-09-21T09:23:05.654 INFO:teuthology.task.internal:Duration was 477.486145 seconds 2024-09-21T09:23:05.654 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-09-21T09:23:05.664 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-09-21T09:23:05.664 DEBUG:teuthology.orchestra.run.smithi078:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-21T09:23:05.706 INFO:teuthology.orchestra.run.smithi078.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-21T09:23:05.977 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-09-21T09:23:05.977 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi078.front.sepia.ceph.com 2024-09-21T09:23:05.977 DEBUG:teuthology.orchestra.run.smithi078:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-21T09:23:06.037 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-09-21T09:23:06.038 DEBUG:teuthology.orchestra.run.smithi078:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-21T09:23:06.104 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-09-21T09:23:06.104 DEBUG:teuthology.orchestra.run.smithi078:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-21T09:23:06.247 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-09-21T09:23:06.258 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-09-21T09:23:06.258 DEBUG:teuthology.orchestra.run.smithi078:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-21T09:23:06.282 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-09-21T09:23:06.293 DEBUG:teuthology.orchestra.run.smithi078:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-21T09:23:06.346 INFO:teuthology.orchestra.run.smithi078.stdout:kernel.core_pattern = core 2024-09-21T09:23:06.392 DEBUG:teuthology.orchestra.run.smithi078:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-21T09:23:06.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:23:06.447 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-09-21T09:23:06.458 INFO:teuthology.task.internal:Transferring archived files... 2024-09-21T09:23:06.459 DEBUG:teuthology.misc:Transferring archived files from smithi078:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi/7913639/remote/smithi078 2024-09-21T09:23:06.459 DEBUG:teuthology.orchestra.run.smithi078:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-21T09:23:06.519 INFO:teuthology.task.internal:Removing archive directory... 2024-09-21T09:23:06.520 DEBUG:teuthology.orchestra.run.smithi078:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-21T09:23:06.570 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-09-21T09:23:06.582 INFO:teuthology.task.internal:Not uploading archives. 2024-09-21T09:23:06.582 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-09-21T09:23:06.630 INFO:teuthology.task.internal:Tidying up after the test... 2024-09-21T09:23:06.630 DEBUG:teuthology.orchestra.run.smithi078:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-21T09:23:06.645 INFO:teuthology.orchestra.run.smithi078.stdout: 265324 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 21 09:23 /home/ubuntu/cephtest 2024-09-21T09:23:06.646 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-09-21T09:23:06.657 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-09-21T09:23:06.687 INFO:teuthology.run:Summary data: description: upgrade/cephfs/nofs/{bluestore-bitmap centos_9.stream conf/{client mds mgr mon osd} no-mds-cluster overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn pg_health} tasks/{0-from/reef 1-upgrade}} duration: 477.48614478111267 flavor: default owner: scheduled_teuthology@teuthology success: true 2024-09-21T09:23:06.687 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-21T09:23:06.771 INFO:teuthology.run:pass